From 985a750ce7f7c4d3d9d72c943d99a258f4b13033 Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Thu, 27 Aug 2026 00:23:56 -0400 Subject: [PATCH 01/28] Make the API docs and the discovery document match the server The /api/v1 discovery document listed GET /{owner}/{repo}/api/commit/{sha}/merge-queue, which has no route, and pointed docs at https://git.example.com/api on every host. The endpoint line is gone and docs is now derived from the request base like the other URLs in the document (AGENTS.md section 5: nothing in crates/ knows a hostname). web/API.md section 5 and the API page said every JSON write needs admin. Creating a repository and starting an op need write (admin.rs, ui.rs); deleting a repository and writing policy or settings need admin (admin.rs, policy.rs, settings.rs). The page also had no row for settings and dropped the /api segment from every repository path. API.md, docs/CONTRACT.md and the SDK comment promised an [integrations] settings section that walgit-config rejects; the section 6 checklist gave browser_base as /api/v1 instead of /api-browser/v1; the overview shape pointed at a Go file that does not exist; the bundle design doc still named /services/install.sh. Tests: api_v1 asserts docs is derived from the discovery base, and a new test pins the write gates (a write token gets 403 on PUT and DELETE of policy and settings, an admin token gets 204 and 200). The walgit-config settings test asserts [integrations] is refused. Co-Authored-By: Claude Fable 5 --- crates/walgit-config/src/lib.rs | 2 + crates/walgit-server/src/web/v1.rs | 5 +- crates/walgit-server/tests/api_v1.rs | 98 ++++++++++++++++++++++++++++ docs/BUNDLE_URI_DESIGN.md | 2 +- docs/CONTRACT.md | 2 +- web/API.md | 12 ++-- web/sdk/repos.ts | 2 +- web/src/pages/ApiPage.tsx | 29 ++++---- 8 files changed, 129 insertions(+), 23 deletions(-) diff --git a/crates/walgit-config/src/lib.rs b/crates/walgit-config/src/lib.rs index 9132dc0..d514eb5 100644 --- a/crates/walgit-config/src/lib.rs +++ b/crates/walgit-config/src/lib.rs @@ -1894,6 +1894,8 @@ listen = \"0.0.0.0:1\"\n", .unwrap_err() .to_string(); assert!(e.contains("[server]"), "{e}"); + // A section the docs once promised but the code never accepted. + assert!(base.with_settings("[integrations]\nx = 1\n").is_err()); // Unknown key inside an allowed section. assert!(base.with_settings("[bundles]\nnope = 1\n").is_err()); // Invalid effective config (incremental without a base). diff --git a/crates/walgit-server/src/web/v1.rs b/crates/walgit-server/src/web/v1.rs index 49813f8..211fb8a 100644 --- a/crates/walgit-server/src/web/v1.rs +++ b/crates/walgit-server/src/web/v1.rs @@ -193,7 +193,7 @@ struct Discovery<'a> { base: String, browser_base: String, sdk: String, - docs: &'a str, + docs: String, auth: DiscoveryAuth<'a>, endpoints: Vec<&'a str>, } @@ -216,7 +216,7 @@ async fn discovery(State(st): State>, headers: HeaderMap) -> Respo // D27: non-repo browser lane (popup). Repo JSON is /{o}/{r}/api-browser/*. browser_base: format!("{base_url}{API_BROWSER}/v1"), sdk: format!("{base_url}/repos.js"), - docs: "https://git.example.com/api", + docs: format!("{base_url}/api"), auth: DiscoveryAuth { bearer: "Authorization: Bearer (an access token from /_auth/tokens, a static token, or an ID token)".to_string(), setup: format!("{base_url}/services/setup.json"), @@ -237,7 +237,6 @@ async fn discovery(State(st): State>, headers: HeaderMap) -> Respo "GET /{owner}/{repo}/api/blob/{rev}/{path}[?raw]", "GET /{owner}/{repo}/api/commits?ref&path&skip&n", "GET /{owner}/{repo}/api/commit/{sha}", - "GET /{owner}/{repo}/api/commit/{sha}/merge-queue", "GET /{owner}/{repo}/api/overview", "GET /{owner}/{repo}/api/tasks[/{id}]", "GET /{owner}/{repo}/api/ops", diff --git a/crates/walgit-server/tests/api_v1.rs b/crates/walgit-server/tests/api_v1.rs index db4e3af..5a57363 100644 --- a/crates/walgit-server/tests/api_v1.rs +++ b/crates/walgit-server/tests/api_v1.rs @@ -28,6 +28,24 @@ async fn req( let headers = resp.headers().clone(); Ok((status, resp.text().await?, headers)) } +async fn req_body( + server: &Server, + method: reqwest::Method, + path: &str, + extra: &[(&str, &str)], + body: &'static str, +) -> anyhow::Result<(reqwest::StatusCode, String)> { + let mut r = reqwest::Client::new() + .request(method, format!("{}{path}", server.base_url)) + .header("Accept", "application/json") + .body(body); + for (k, v) in extra { + r = r.header(*k, *v); + } + let resp = r.send().await?; + let status = resp.status(); + Ok((status, resp.text().await?)) +} fn hdr(h: &reqwest::header::HeaderMap, k: &str) -> String { h.get(k) .and_then(|v| v.to_str().ok()) @@ -100,6 +118,14 @@ async fn v1_surface_and_browser_lane() -> TestResult { .unwrap() .ends_with("/api-browser/v1/authenticate") ); + // `docs` is this host's API page, derived from the same base as every + // other URL in the document (AGENTS.md §5: no hardcoded hostnames). + let base = d["base"].as_str().unwrap(); + assert_eq!( + d["docs"], + format!("{}/api", base.trim_end_matches("/api/v1")), + "{d}" + ); // me (auth mode none in tests → anonymous principal) let (st, _, h) = req(&server, reqwest::Method::GET, "/api/v1/me", &[]).await?; @@ -532,3 +558,75 @@ async fn repository_delete_requires_admin() -> TestResult { ); Ok(()) } + +/// D24 and API.md §5: on the JSON surface a write token creates repositories, +/// but the policy and settings documents move only with admin. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn policy_and_settings_writes_require_admin() -> TestResult { + let server = Server::start_with_tweak(|c| { + c.server.auth.mode = walgit_config::AuthMode::Token; + c.server.auth.anonymous_read = false; + c.server.auth.tokens = vec![ + walgit_config::StaticToken { + principal: "writer".into(), + token: "writer-token".into(), + token_env: None, + write: true, + admin: false, + }, + walgit_config::StaticToken { + principal: "admin".into(), + token: "admin-token".into(), + token_env: None, + write: true, + admin: true, + }, + ]; + }) + .await?; + let writer = [("Authorization", "Bearer writer-token")]; + let admin = [("Authorization", "Bearer admin-token")]; + assert_eq!( + req(&server, reqwest::Method::PUT, "/gates/repo/api", &writer) + .await? + .0, + 201, + "write permission creates the repository" + ); + + const POLICY: &str = r#"{"version":1,"groups":[],"rules":[]}"#; + const SETTINGS: &str = "[bundles]\nmin_commits = 3\n"; + for (path, body) in [ + ("/gates/repo/api/policy", POLICY), + ("/gates/repo/api/settings", SETTINGS), + ] { + let (st, text) = req_body(&server, reqwest::Method::PUT, path, &writer, body).await?; + assert_eq!(st, 403, "a write token must not PUT {path}: {text}"); + assert_eq!( + req(&server, reqwest::Method::DELETE, path, &writer) + .await? + .0, + 403, + "a write token must not DELETE {path}" + ); + } + let (st, text) = req_body( + &server, + reqwest::Method::PUT, + "/gates/repo/api/policy", + &admin, + POLICY, + ) + .await?; + assert_eq!(st, 204, "{text}"); + let (st, text) = req_body( + &server, + reqwest::Method::PUT, + "/gates/repo/api/settings", + &admin, + SETTINGS, + ) + .await?; + assert_eq!(st, 200, "{text}"); + Ok(()) +} diff --git a/docs/BUNDLE_URI_DESIGN.md b/docs/BUNDLE_URI_DESIGN.md index 8b442dc..9b0a570 100644 --- a/docs/BUNDLE_URI_DESIGN.md +++ b/docs/BUNDLE_URI_DESIGN.md @@ -71,7 +71,7 @@ A git bundle file = header + packfile. | **Naming / storage** | `bundles//-.bundle` (immutable, content-addressed, ETag = checksum), list at `bundles/list.pb` (CAS). Keys never overwritten. | Immutable → `Cache-Control: immutable`, Range, CDN; CAS'd list = atomic publish. | | **Serving** | `serve_via = proxy` (streamed from the bucket through a serving host with the full static contract: ETag/304/If-Range/Range/HEAD) or `signed_url` (direct object-store URL). Signing may be unavailable or denied by the store; failure falls back to proxy per entry and never fails the listing. | Static contract either way; direct URLs remove the serving process from the byte path when the store permits signing. | | **Two lists: clone and catch-up** (2026-08-22) | `bundles/list` is the clone list (fulls + chain); **`bundles/catchup`** is the same list **without the fulls**, and it is what every recipe records in `fetch.bundleURI`. Dailies chain *through* the weekly: Sunday's daily and the weekly fire at the same instant and have the same tips, so Monday's daily is cut on Sunday's daily (tie → own chain, `slots::base_for_incremental`), and retention keeps the chain under every kept full (`keep = 2` on the weekly = two weeks of catch-up through bundles). | git's creationToken walk goes newest-first and a full has no prerequisites, so a fetching client downloads **every full newer than its token** — the new weekly, 32 GB from a large repository, on the first fetch after Sunday (measured on the rig: round 1 of `rig/catchup`). A client with history never needs a full; with no fulls in its list and a chain that crosses the week, it walks daily → daily to a link whose prerequisites it has. Fresh clones still take the newest weekly (they have its objects, so Monday's prerequisites hold). e2e `fetch_after_the_recipe_clone_uses_the_bundles` covers the rollover. | -| **Advertising** | v2 capability `bundle-uri` + the `bundle-uri` command; static list at `/{o}/{r}.git/bundles/list`; `/services/install.sh` sets `transfer.bundleURI=true` + `fetch.bundleURI`. The narrated fetch echoes each advertised bundle: `* bundle-uri: /acme/monorepo/bundles/weekly/ (32.3 GB, full, seq 1, token …)`. | Users see where bytes come from. | +| **Advertising** | v2 capability `bundle-uri` + the `bundle-uri` command; static list at `/{o}/{r}.git/bundles/list`; `/services/public/install.sh` sets `transfer.bundleURI=true` + `fetch.bundleURI`. The narrated fetch echoes each advertised bundle: `* bundle-uri: /acme/monorepo/bundles/weekly/ (32.3 GB, full, seq 1, token …)`. | Users see where bytes come from. | | **Forcing** | `bundles.require = ["acme/monorepo"]` (D17): an **unbounded zero-have** fetch (a full clone that skipped bundles) is refused with the exact fix; `--depth`/`--filter` zero-have fetches (CI) and all fetches with haves proceed. **One-shot fallback** (2026-08-21): a principal that fetched `bundles/list` within the hour *tried* bundle-uri — git does not retry a failed bundle download and then sends exactly this zero-have fetch — so it gets one upload-pack clone per 6 h with a loud band-2 warning; the next one and anyone who never tried are refused, truthfully. | Protects the instances from the one request they cannot serve; keeps CI's shallow/partial clones (the 2075 s benchmark shape) on upload-pack, where they take ~8 s. The fallback trades ≈ 32 GB of egress + minutes of the SSD host (deltas reused from the base: pack-objects is I/O-bound, not CPU-bound) for "`git clone` never fails"; the rate limit keeps a fleet of misconfigured clients from turning the SSD host into a 32 GB-per-clone server — the same request without the list fetch first is still refused. | ## 4. Scheduling: calendar slots with backfill diff --git a/docs/CONTRACT.md b/docs/CONTRACT.md index 1825871..12093f1 100644 --- a/docs/CONTRACT.md +++ b/docs/CONTRACT.md @@ -25,7 +25,7 @@ Read `AGENTS.md` first (design §1–§2, decisions §3; the original layout/pha `ObjectStoreExt`, `Prefixed`, `memory::MemoryStore`, `util::{collect,once,file_stream,backoff,retry}`), placeholder modules `coord.rs`, `gcs.rs`, `s3.rs`. - `walgit-config`: `Config` for walgit.toml (+ `WALGIT__` env overrides, `PORT`); `Config::with_settings` accepts - only `[bundles]`, `[maintenance]`, `[compaction]`, `[upstream]`, and `[integrations]` in repo-scoped settings. + only `[bundles]`, `[maintenance]`, `[compaction]` and `[upstream]` in repo-scoped settings. ## walgit-git (owner: GitEngine) diff --git a/web/API.md b/web/API.md index 9239f1e..d0c27f4 100644 --- a/web/API.md +++ b/web/API.md @@ -265,7 +265,7 @@ removes it (admin permission) — the same handlers as `PUT|DELETE /{owner}/{rep `GET|PUT|DELETE …/policy` is the push policy document (`docs/POLICY.md`). `GET|PUT|DELETE /{o}/{r}/api/settings` (D24, 2026-08-21) is the repository's **settings in the WAL**: a TOML document -restricted to `[bundles]`, `[maintenance]`, `[compaction]`, `[upstream]`, and `[integrations]`, merged over the +restricted to `[bundles]`, `[maintenance]`, `[compaction]` and `[upstream]`, merged over the host's config (`effective config`). `GET` → `{revision, author, updated_at, message, toml}` (`revision: 0` = none). `PUT` body = the TOML (`?message=` optional), validated against the serving host's build — 400 with the reason and nothing published @@ -464,7 +464,7 @@ list by `commit_date` day and shows `subject` + `author`. Backs the "WAL" tab. Not needed by Code/Commits pages; a host without a WAL should return `404` (the tab then shows the error text). Shape is in -`api.ts#Overview` / `overview.go`: `repo`, `clone_url`, `hostname`, +`api.ts#Overview` / `struct Overview` in `crates/walgit-server/src/web/ui.rs`: `repo`, `clone_url`, `hostname`, `health{status: ok|degraded|error, issues[], deep, suggestions[{op, params?, reason, auto?}]}` — `deep` is the last connectivity audit as recorded in the store (`fsck.pb`, any maintainer), `auto` says how/when the maintainer loop performs a suggestion by itself (absent = a human must) — `manifest{version, @@ -505,14 +505,16 @@ redelivers). Never cached, never served to the SPA. sha-addressed JSON in an LRU, since it can never go stale. - Reads must be as fresh as a `git fetch` from the same host would be: after a push is acknowledged, the next API call (any node) reflects it. -- Writes on the JSON surface are admin only: `PUT|DELETE /{o}/{r}/api`, - `PUT|DELETE …/policy`, `POST …/ops/{op}`. Content moves over git +- Writes on the JSON surface need a token with the matching permission: + write for `PUT /{o}/{r}/api` (create) and `POST …/ops/{op}`, admin for + `DELETE /{o}/{r}/api`, `PUT|DELETE …/policy` and `PUT|DELETE …/settings` + (D24: write is push, not admin). Content moves over git (`git-receive-pack`) and LFS, never through JSON. ## 6. Minimal conformance checklist ``` -GET /api/v1 → 200 {version:1, base, browser_base=/api/v1, sdk, auth, endpoints} +GET /api/v1 → 200 {name, version:1, base, browser_base=/api-browser/v1, sdk, docs, auth, endpoints} GET /api/v1/me → 200 {principal,write,anonymous} | 401; no-store GET /api/v1/owners → 200 [..] ([] when empty) GET /api/v1/owners/nobody/repos → 200 [] diff --git a/web/sdk/repos.ts b/web/sdk/repos.ts index c93b2a7..1dbdee8 100644 --- a/web/sdk/repos.ts +++ b/web/sdk/repos.ts @@ -714,7 +714,7 @@ export class RepoClient { }), }; - /** D24: WAL-backed TOML overrides of [bundles], [maintenance], [compaction], [upstream], and [integrations]. */ + /** D24: WAL-backed TOML overrides of [bundles], [maintenance], [compaction] and [upstream]. */ readonly settings = { /** The settings document (`revision: 0` = none). */ get: (opts?: CallOptions) => this.client.json(`${this.p}/settings`, opts), diff --git a/web/src/pages/ApiPage.tsx b/web/src/pages/ApiPage.tsx index 990f211..5361efd 100644 --- a/web/src/pages/ApiPage.tsx +++ b/web/src/pages/ApiPage.tsx @@ -78,7 +78,7 @@ export function ApiPage() { - + @@ -97,42 +97,47 @@ export function ApiPage() { desc={<>Repo summary: {`{owner,name,full_name,head,branches,tags,clone_url,html_url,api_url}`} (O(1) ref counts). PUT creates (write), DELETE removes (admin).} cache="SWR + ETag" /> - Default branch only: {`{head:{name,sha}|null}`}. O(1) whatever the ref count.} cache="SWR + ETag" /> + Default branch only: {`{head:{name,sha}|null}`}. O(1) whatever the ref count.} cache="SWR + ETag" /> One name-sorted page {`{refs:[{name,sha}],more}`}; tags peeled; n ≤ 1000. With Accept: text/event-stream: one ref event per match as found.} cache="SWR" /> Splits a GitHub-shaped ref/path into {`{ref,sha,path,kind}`}; longest existing branch/tag wins, then a revision. Do this once, then address by sha.} cache="SWR + ETag" /> Directory listing {`{entries:[{name,type,mode,size,sha}],commit?,readme?}`}, dirs first, with the latest commit touching the path and README contents.} cache="sha → immutable · name → SWR + ETag" /> {`{name,size,contents}`} or binary:true / too_large:true; ?raw returns the bytes as text/plain.} cache="sha → immutable · name → SWR + ETag" /> History page {`{commits:[Commit],more}`}, optionally for one path; n ≤ 200; paginate with skip += commits.length.} cache="sha → immutable · name → SWR + ETag" /> {`{commit,stats:[{path,additions,deletions}],patch}`} — unified diff against the first parent; any revision accepted.} cache="full sha → immutable · else SWR + ETag" /> - Push policy document (GET/PUT/DELETE, write).} cache="no-store" /> - + Push policy document (GET read; PUT/DELETE admin).} cache="no-store" /> What the answering instance is doing to the repo ({`{hostname,running,recent}`}); attach to a task or start a maintenance op as an SSE stream.} + path={`/${r}/api/settings`} + desc={<>Per-repository settings in the WAL (GET read; PUT/DELETE admin); also /effective, /history, /describe and POST /validate.} + cache="no-store" + /> + + What the answering instance is doing to the repo ({`{hostname,running,recent}`}); attach to a task or start a maintenance op (write) as an SSE stream.} cache="no-store" /> From 366410f8015f7a6c756a29441504a8b6e521f28a Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Thu, 27 Aug 2026 00:24:00 -0400 Subject: [PATCH 02/28] Let a static token entry name only token_env StaticToken.token at crates/walgit-config/src/lib.rs:212 carried no serde default while token_env on the next line did, so an entry that named only an environment variable failed to deserialize with "missing field token". Every shipped example writes that shape, including README.md:19, walgit.example.toml:48 and walgit.standalone.toml:36, so the quick start config was rejected by "walgit --config walgit.toml config check". The field now defaults to the empty string, and Config::validate at lib.rs:1491 still refuses an entry that names neither token nor token_env, so a config with no way in stays fail-closed as GOAL.md asks. tests::auth_modes_validate_fail_closed parses the README entry and asserts the entry with neither form still errors; it fails on the old struct with the same TOML parse error the binary printed. Co-Authored-By: Claude Fable 5 --- crates/walgit-config/src/lib.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/walgit-config/src/lib.rs b/crates/walgit-config/src/lib.rs index 9132dc0..069a924 100644 --- a/crates/walgit-config/src/lib.rs +++ b/crates/walgit-config/src/lib.rs @@ -209,6 +209,7 @@ pub enum AuthMode { pub struct StaticToken { pub principal: String, /// Read from env var if set, else literal. + #[serde(default)] pub token: String, #[serde(default)] pub token_env: Option, @@ -1938,6 +1939,25 @@ audiences = ["walgit-cli", "https://git.example.com"] let err = Config::parse("[store]\nbucket = \"b\"\n[server.auth]\nmode = \"token\"\n") .unwrap_err(); assert!(err.to_string().contains("tokens"), "{err}"); + // `token_env` alone is a whole entry: the README quick start writes exactly this. + let readme = Config::parse( + "[store]\nbucket = \"b\"\n[server.auth]\nmode = \"token\"\ntokens = [{ principal = \"me\", token_env = \"WALGIT_TOKEN_ME\", write = true }]\n", + ) + .unwrap(); + let t = &readme.server.auth.tokens[0]; + assert_eq!(t.principal, "me"); + assert!(t.token.is_empty(), "{:?}", t.token); + assert_eq!(t.token_env.as_deref(), Some("WALGIT_TOKEN_ME")); + assert!(t.write && !t.admin); + // Neither key names a secret, so the entry could never let anyone in. + let err = Config::parse( + "[store]\nbucket = \"b\"\n[server.auth]\nmode = \"token\"\ntokens = [{ principal = \"me\", write = true }]\n", + ) + .unwrap_err(); + assert!( + err.to_string().contains("needs `token` or `token_env`"), + "{err}" + ); // oidc: anonymous_read off, an allowlist, and a way in. let err = Config::parse("[store]\nbucket = \"b\"\n[server.auth]\nmode = \"oidc\"\nanonymous_read = false\nallowed_domains = [\"example.com\"]\n").unwrap_err(); assert!(err.to_string().contains("way in"), "{err}"); From a21a082357ab8047ad20a9037c1c0c846d196d4b Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Thu, 27 Aug 2026 00:24:23 -0400 Subject: [PATCH 03/28] Fix the config check invocation in walgit.example.toml The header comment at walgit.example.toml:6 said "walgit config check walgit.toml", but the check subcommand takes no positional argument and running it that way prints "error: unexpected argument 'walgit.toml' found" and exits 2. The path comes from the global --config flag declared at crates/walgit-cli/src/lib.rs:42-48, so the comment now reads "walgit --config walgit.toml config check". I ran that form against walgit.example.toml and walgit.standalone.toml with the built binary and both print "config OK". Co-Authored-By: Claude Fable 5 --- walgit.example.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/walgit.example.toml b/walgit.example.toml index 3c4371d..ea161d4 100644 --- a/walgit.example.toml +++ b/walgit.example.toml @@ -3,7 +3,7 @@ # Start from walgit.standalone.toml for a first run; come here when you need a key. # Normative bundle-slot semantics: docs/BUNDLE_URI_DESIGN.md §4. # Every key can also be set from the environment: WALGIT__SECTION__KEY=value (TOML value syntax). -# Validate with: walgit config check walgit.toml +# Validate with: walgit --config walgit.toml config check [server] listen = "127.0.0.1:8080" # default; `mode = none` is refused unless this is loopback. Public bind: 0.0.0.0 with token/oidc. From 6f046e33a9043da6f98495ebc657b37b1ea29edd Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Thu, 27 Aug 2026 00:24:59 -0400 Subject: [PATCH 04/28] Say mode none grants admin as well as write in the README The auth mode table at README.md:143 described mode none as everyone being anon with write, which understates what the mode does. AuthMode::None returns a principal with write true and admin true at crates/walgit-server/src/auth.rs:672-677, and AGENTS.md:78 states the same in the security contract, so a reader of the README alone would not know that a loopback run also hands out settings and policy.json writes. The row now says write and admin. This is a documentation correction with no code change, so the code cited above is the proof. Co-Authored-By: Claude Fable 5 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 049bc60..5008ca5 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ each repository one maintainer (placement globs) and you are done. | mode | who gets in | how git authenticates | |---|---|---| -| `none` | everyone is `anon` with write — loopback experiments | nothing | +| `none` | everyone is `anon` with write and admin — loopback experiments | nothing | | `token` | static `tokens` in the config (`token_env` reads the secret from the environment) | `Authorization: Bearer `, or the token as an HTTP Basic password | | `oidc` | any OpenID Connect issuer (`issuer`, `oauth_client_id/secret`, `allowed_domains`/`allowed_emails`): Google, Entra, Okta, Auth0, Keycloak, Dex, GitLab… | a **walgit access token**: sign in once in the browser, create one at `/_auth/tokens`, paste it into the installer. Stateless (HMAC with `session_secret`, `access_token_ttl`); rotating the secret revokes all. ID tokens from the issuer (`audiences`) and static `tokens` work too. | From 386cc95c1f8eab2d9ae809bd72ce76658dc1b9c4 Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Thu, 27 Aug 2026 00:26:06 -0400 Subject: [PATCH 05/28] Read the forwarded client credential only when an edge announced it client_authorization in crates/walgit-server/src/auth.rs:897 read X-Walgit-Authorization before it checked anything else, so a client talking to walgit directly could put a credential in that header and have it taken as the client credential. AGENTS.md D39 (2) and section 1.3 say the forwarded copy counts only when an edge announced client-authorization in X-Walgit-Capabilities on that request, and that nothing is assumed when walgit is hit with nothing in front of it. The function now consults edge_owns_authorization first and returns the plain Authorization header value when no capability was announced, reaching for the forwarded copy only behind an announcing edge. Behaviour behind an edge is unchanged, a missing forwarded copy there still meaning the client sent no credential. The unit test edge_owned_authorization_is_not_the_client now covers the direction that was missing: Authorization Bearer a with X-Walgit-Authorization Bearer b and no capability header yields a, and the same two headers with the capability yield b. Co-Authored-By: Claude Fable 5 --- crates/walgit-server/src/auth.rs | 45 +++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/crates/walgit-server/src/auth.rs b/crates/walgit-server/src/auth.rs index 78d4312..05cacdb 100644 --- a/crates/walgit-server/src/auth.rs +++ b/crates/walgit-server/src/auth.rs @@ -894,24 +894,24 @@ fn edge_owns_authorization(headers: &HeaderMap) -> bool { }) } -/// The client's `Authorization` header value (edge-forwarded copy first). +/// The client's `Authorization` header value: the header itself when walgit is hit +/// directly, the edge-forwarded copy when an edge announced `client-authorization`. fn client_authorization(headers: &HeaderMap) -> Option { - if let Some(v) = headers - .get(FORWARDED_AUTHORIZATION_HEADER) - .and_then(|v| v.to_str().ok()) - .map(str::trim) - .filter(|v| !v.is_empty()) - { - return Some(v.to_string()); + // Nothing announced the capability, so `Authorization` is the client's own and a + // forwarded copy nobody vouched for is not read at all (D39 (2), §1.3). + if !edge_owns_authorization(headers) { + return headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .map(str::to_string); } // Behind the edge, a missing copy means the client sent no credential; the // Authorization that is there is the hop's own. - if edge_owns_authorization(headers) { - return None; - } headers - .get(axum::http::header::AUTHORIZATION) + .get(FORWARDED_AUTHORIZATION_HEADER) .and_then(|v| v.to_str().ok()) + .map(str::trim) + .filter(|v| !v.is_empty()) .map(str::to_string) } @@ -1002,7 +1002,8 @@ mod tests { /// Behind the edge (`client-authorization` capability) `Authorization` is the hop's own /// credential: with no `X-Walgit-Authorization` there is no client bearer (so the session - /// cookie gets its turn). Without the capability, `Authorization` is the client's. + /// cookie gets its turn). Without the capability, `Authorization` is the client's and the + /// forwarded header is not read at all. #[test] fn edge_owned_authorization_is_not_the_client() { let mut h = HeaderMap::new(); @@ -1018,6 +1019,24 @@ mod tests { "Bearer client".parse().unwrap(), ); assert_eq!(bearer_token(&h).as_deref(), Some("client")); + + let mut direct = HeaderMap::new(); + direct.insert(AUTHORIZATION, "Bearer a".parse().unwrap()); + direct.insert(FORWARDED_AUTHORIZATION_HEADER, "Bearer b".parse().unwrap()); + assert_eq!( + bearer_token(&direct).as_deref(), + Some("a"), + "hit directly, a forwarded copy no edge announced is ignored" + ); + direct.insert( + crate::static_object::CAPABILITIES_HEADER, + "client-authorization".parse().unwrap(), + ); + assert_eq!( + bearer_token(&direct).as_deref(), + Some("b"), + "the announced capability makes the forwarded copy the client's" + ); } #[test] From 4bb3ce63d74e929c14432c567e1c0e2f14bcf116 Mon Sep 17 00:00:00 2001 From: Ethan Stoner Date: Thu, 27 Aug 2026 01:15:28 -0700 Subject: [PATCH 06/28] Classify transient S3 failures as retryable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The S3 backend turned every failure that was not a precondition failure into `StoreError::Other`, so nothing on the S3 write path could ever produce `StoreError::Retryable`. GCS classifies the same conditions — `gcs::is_retryable` covers Unavailable/DeadlineExceeded/ResourceExhausted/ Internal/Aborted, HTTP 503/504/429/500, and connect/IO faults — which made this a "GCS only" behaviour of the kind AGENTS.md rules out. Two readers of `StoreError::is_retryable` were affected: - `coord::cas_update`, the manifest CAS that is the only commit point. On `Retryable` it sleeps with jittered backoff and re-reads; on anything else it returns `CoordError::Store` and gives up. A throttled manifest PUT therefore failed the push outright on S3. The SDK's own retries (standard mode, three attempts) run underneath this and do not replace it — what reaches walgit is what the SDK could not absorb. - `smart::wal_err`, which maps retryable store errors to 503 and everything else to 500. On S3 a transient bucket fault reached the git client as a hard 500 instead of the 503 its comment describes. Add `is_retryable` for `SdkError` — dispatch failures and timeouts, the transient AWS error codes, and the transient HTTP statuses for S3-compatible stores that do not use AWS codes — and route every `SdkError` site through it: put, list, head, delete, and the multipart create/upload/copy/complete/abort paths. Tested with a fake S3 bound on an ephemeral port, driving real `SdkError` values through the classifiers with SDK retries disabled: throttling, a server fault, an unrecognised transient status, and an unreachable endpoint are retryable; access denied is not; a failed precondition stays a failed precondition. --- crates/walgit-store/src/s3.rs | 268 +++++++++++++++++++++++++++++++--- 1 file changed, 250 insertions(+), 18 deletions(-) diff --git a/crates/walgit-store/src/s3.rs b/crates/walgit-store/src/s3.rs index 2877ae2..a4b31b4 100644 --- a/crates/walgit-store/src/s3.rs +++ b/crates/walgit-store/src/s3.rs @@ -241,6 +241,68 @@ where err.as_service_error().map(|e| e.meta().code()).flatten() } +/// S3 error codes that mean "the service could not serve this request now", +/// as opposed to "this request is wrong". The GCS counterpart is +/// `gcs::is_retryable`'s status set. +fn is_transient_code(code: &str) -> bool { + matches!( + code, + // Throttling: the request rate exceeded what the prefix will take. + "SlowDown" | "RequestLimitExceeded" | "ThrottlingException" | "TooManyRequests" + // The service's own faults. + | "InternalError" | "ServiceUnavailable" + // The socket went idle mid-PUT; S3 reports this as 400 RequestTimeout. + | "RequestTimeout" + ) +} + +/// HTTP statuses that mean the same, for services whose error codes we do not +/// recognise (rustfs and other S3-compatible stores do not all use AWS codes). +fn is_transient_status(status: u16) -> bool { + matches!(status, 429 | 500 | 502 | 503 | 504) +} + +/// Whether an SDK failure is worth another attempt at walgit's layer. +/// +/// The SDK retries transient failures itself (standard mode, three attempts) +/// and surfaces what it could not absorb — but those leftovers are still +/// transient, and walgit has its own, longer-horizon retry above them: +/// `coord::cas_update` backs off and re-reads the manifest on `Retryable`, and +/// `smart::wal_err` turns it into a 503 the git client can retry rather than a +/// 500 it cannot. Everything here used to collapse into `Other`, so a +/// throttled manifest CAS failed the push outright on S3 while the same +/// throttle on GCS was absorbed. +fn is_retryable(err: &aws_sdk_s3::error::SdkError) -> bool +where + E: aws_sdk_s3::error::ProvideErrorMetadata, +{ + // No response at all: a timeout or a connection that never landed. + if matches!( + err, + aws_sdk_s3::error::SdkError::TimeoutError(_) + | aws_sdk_s3::error::SdkError::DispatchFailure(_) + ) { + return true; + } + err_code(err).is_some_and(is_transient_code) + || err + .raw_response() + .is_some_and(|r| is_transient_status(r.status().as_u16())) +} + +/// Wrap an SDK failure, keeping the retryable/permanent distinction that +/// `StoreError::is_retryable` is read for. +fn classify_error(context: &str, err: aws_sdk_s3::error::SdkError) -> StoreError +where + E: aws_sdk_s3::error::ProvideErrorMetadata + std::error::Error + Send + Sync + 'static, +{ + if is_retryable(&err) { + StoreError::Retryable(anyhow::anyhow!("{context}: {err}")) + } else { + StoreError::Other(anyhow::anyhow!("{context}: {err}")) + } +} + fn classify_put_error( key: &str, err: aws_sdk_s3::error::SdkError, @@ -251,14 +313,14 @@ fn classify_put_error( key: key.into(), current: None, }, - _ => StoreError::Other(anyhow::anyhow!("s3 put error: {err}")), + _ => classify_error("s3 put error", err), } } fn classify_list_error( err: aws_sdk_s3::error::SdkError, ) -> StoreError { - StoreError::Other(anyhow::anyhow!("s3 list error: {err}")) + classify_error("s3 list error", err) } #[async_trait::async_trait] @@ -297,7 +359,7 @@ impl ObjectStore for S3Store { { return Ok(None); } - Err(StoreError::Other(anyhow::anyhow!("s3 head error: {err}"))) + Err(classify_error("s3 head error", err)) } } } @@ -402,7 +464,7 @@ impl ObjectStore for S3Store { return Ok(()); } } - Err(StoreError::Other(anyhow::anyhow!("s3 delete error: {err}"))) + Err(classify_error("s3 delete error", err)) } } } @@ -587,7 +649,7 @@ impl ObjectStore for S3Store { let upload = create .send() .await - .map_err(|e| StoreError::Other(anyhow::anyhow!("s3 create multipart: {e}")))?; + .map_err(|e| classify_error("s3 create multipart", e))?; let upload_id = upload .upload_id() .ok_or_else(|| { @@ -626,9 +688,7 @@ impl ObjectStore for S3Store { .copy_source_range(format!("bytes={from}-{}", from + len - 1)) .send() .await - .map_err(|e| { - StoreError::Other(anyhow::anyhow!("s3 upload part copy: {e}")) - })?; + .map_err(|e| classify_error("s3 upload part copy", e))?; let etag = part .copy_part_result() .and_then(|r| r.e_tag()) @@ -681,7 +741,7 @@ impl ObjectStore for S3Store { .content_length(len as i64) .send() .await - .map_err(|e| StoreError::Other(anyhow::anyhow!("s3 upload part: {e}")))?; + .map_err(|e| classify_error("s3 upload part", e))?; parts.push( aws_sdk_s3::types::CompletedPart::builder() .e_tag(part.e_tag().unwrap_or("").to_owned()) @@ -715,9 +775,7 @@ impl ObjectStore for S3Store { Ok(r) => r, Err(e) => { let _ = self.abort_multipart(dest, &upload_id).await; - return Err(StoreError::Other(anyhow::anyhow!( - "s3 complete multipart: {e}" - ))); + return Err(classify_error("s3 complete multipart", e)); } }; let etag = resp.e_tag().map(|s| s.trim_matches('"').to_owned()); @@ -777,7 +835,7 @@ impl S3Store { let upload = create .send() .await - .map_err(|e| StoreError::Other(anyhow::anyhow!("s3 create multipart: {e}")))?; + .map_err(|e| classify_error("s3 create multipart", e))?; let upload_id = upload .upload_id() @@ -835,7 +893,7 @@ impl S3Store { Ok(p) => p, Err(e) => { let _ = self.abort_multipart(key, &upload_id).await; - return Err(StoreError::Other(anyhow::anyhow!("s3 upload part: {e}"))); + return Err(classify_error("s3 upload part", e)); } }; @@ -868,9 +926,7 @@ impl S3Store { Ok(r) => r, Err(e) => { let _ = self.abort_multipart(key, &upload_id).await; - return Err(StoreError::Other(anyhow::anyhow!( - "s3 complete multipart: {e}" - ))); + return Err(classify_error("s3 complete multipart", e)); } }; @@ -890,7 +946,7 @@ impl S3Store { .upload_id(upload_id) .send() .await - .map_err(|e| StoreError::other(anyhow::anyhow!("abort multipart: {e}")))?; + .map_err(|e| classify_error("abort multipart", e))?; Ok(()) } } @@ -922,6 +978,182 @@ fn static_credentials( mod tests { use super::*; + use aws_sdk_s3::error::SdkError; + use aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Error; + use aws_sdk_s3::operation::put_object::PutObjectError; + + /// A fake S3 that answers every request with one status and error code. + /// Bound on an ephemeral port; the accept loop dies with the test runtime. + async fn fake_s3(status: u16, code: &'static str) -> S3Client { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + tokio::spawn(async move { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mut buf = [0u8; 8192]; + let _ = sock.read(&mut buf).await; + let body = format!( + "{code}fake" + ); + let resp = format!( + "HTTP/1.1 {status} Fake\r\nContent-Type: application/xml\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = sock.write_all(resp.as_bytes()).await; + }); + } + }); + client_for(&format!("http://127.0.0.1:{port}")) + } + + /// SDK retries are disabled so each test observes exactly the error the + /// service produced; walgit's own retry layer is what these tests cover. + fn client_for(endpoint: &str) -> S3Client { + let conf = aws_sdk_s3::config::Config::builder() + .region(aws_sdk_s3::config::Region::new("us-east-1")) + .credentials_provider(static_credentials("test", "test", None)) + .endpoint_url(endpoint) + .force_path_style(true) + .retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled()) + .behavior_version_latest() + .build(); + S3Client::from_conf(conf) + } + + async fn put_error(client: &S3Client) -> SdkError { + client + .put_object() + .bucket("b") + .key("k") + .body(S3ByteStream::from_static(b"x")) + .send() + .await + .expect_err("the fake service fails every request") + } + + async fn list_error(client: &S3Client) -> SdkError { + client + .list_objects_v2() + .bucket("b") + .send() + .await + .expect_err("the fake service fails every request") + } + + #[tokio::test] + async fn throttling_is_retryable() { + let client = fake_s3(503, "SlowDown").await; + assert!(matches!( + classify_put_error("k", put_error(&client).await), + StoreError::Retryable(_) + )); + } + + #[tokio::test] + async fn server_fault_is_retryable() { + let client = fake_s3(500, "InternalError").await; + assert!(matches!( + classify_put_error("k", put_error(&client).await), + StoreError::Retryable(_) + )); + } + + #[tokio::test] + async fn a_transient_status_without_a_known_code_is_retryable() { + let client = fake_s3(504, "SomethingUnrecognised").await; + assert!(matches!( + classify_put_error("k", put_error(&client).await), + StoreError::Retryable(_) + )); + } + + #[tokio::test] + async fn an_unreachable_endpoint_is_retryable() { + // Nothing listens on port 1: a dispatch failure, no response at all. + let client = client_for("http://127.0.0.1:1"); + assert!(matches!( + classify_put_error("k", put_error(&client).await), + StoreError::Retryable(_) + )); + } + + #[tokio::test] + async fn denied_is_permanent() { + let client = fake_s3(403, "AccessDenied").await; + assert!(matches!( + classify_put_error("k", put_error(&client).await), + StoreError::Other(_) + )); + } + + #[tokio::test] + async fn a_failed_precondition_stays_a_failed_precondition() { + let client = fake_s3(412, "PreconditionFailed").await; + assert!(matches!( + classify_put_error("k", put_error(&client).await), + StoreError::PreconditionFailed { .. } + )); + } + + #[tokio::test] + async fn a_throttled_list_is_retryable() { + let client = fake_s3(503, "SlowDown").await; + assert!(matches!( + classify_list_error(list_error(&client).await), + StoreError::Retryable(_) + )); + } + + #[tokio::test] + async fn a_denied_list_is_permanent() { + let client = fake_s3(403, "AccessDenied").await; + assert!(matches!( + classify_list_error(list_error(&client).await), + StoreError::Other(_) + )); + } + + #[test] + fn transient_codes_are_recognised() { + for code in [ + "SlowDown", + "InternalError", + "ServiceUnavailable", + "RequestTimeout", + "RequestLimitExceeded", + "ThrottlingException", + "TooManyRequests", + ] { + assert!(is_transient_code(code), "{code} should be transient"); + } + } + + #[test] + fn permanent_codes_are_not_transient() { + for code in [ + "AccessDenied", + "NoSuchBucket", + "NoSuchKey", + "PreconditionFailed", + "InvalidAccessKeyId", + "EntityTooLarge", + ] { + assert!(!is_transient_code(code), "{code} should be permanent"); + } + } + + #[test] + fn transient_statuses_are_recognised() { + for status in [429, 500, 502, 503, 504] { + assert!(is_transient_status(status), "{status} should be transient"); + } + for status in [400, 403, 404, 409, 412] { + assert!(!is_transient_status(status), "{status} should be permanent"); + } + } + #[test] fn static_credentials_include_session_token_when_present() { let creds = static_credentials("access", "secret", Some("session".into())); From 747513904f40fc39d87d5ebd998657bb344e0c6f Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Thu, 27 Aug 2026 12:37:31 -0400 Subject: [PATCH 07/28] Install libprotobuf-dev in the Containerfile build stage The Rust stage installed protobuf-compiler, which provides protoc but not the well-known .proto files that walgit/v1/wal.proto imports, so the walgit-proto build script failed with "google/protobuf/timestamp.proto: File not found" and the image never built (#21). On Debian those files ship in libprotobuf-dev. jeonck checked both packages in the stage's own base image; the image has not been rebuilt on this machine. Co-Authored-By: Claude Fable 5 --- Containerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Containerfile b/Containerfile index 9ead156..72d74ae 100644 --- a/Containerfile +++ b/Containerfile @@ -24,7 +24,7 @@ RUN pnpm run build && test -f dist/index.html && test -f dist/repos.js # ---- 2. rust build ------------------------------------------------------------------------ FROM docker.io/library/rust:1.97-bookworm AS build -RUN apt-get update && apt-get install -y --no-install-recommends protobuf-compiler pkg-config cmake perl python3 \ +RUN apt-get update && apt-get install -y --no-install-recommends protobuf-compiler libprotobuf-dev pkg-config cmake perl python3 \ && rm -rf /var/lib/apt/lists/* WORKDIR /src COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ From da0b3c080593597fcaec1806a1e2db0e621e79e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20L=C3=BCtke?= Date: Sat, 5 Sep 2026 18:05:13 +0000 Subject: [PATCH 08/28] Ignore local reports --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0908cf0..2f18fa3 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ target/ # local review dumps (findings, smoketests, notes — not for the repo) .review/ +reports/ # built web assets (embedded into the binary at build time; `just web-build`) web/dist/ From 81dc7cb462809c03c4273c4ec8d5775927e278c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20L=C3=BCtke?= Date: Sat, 5 Sep 2026 18:58:55 +0000 Subject: [PATCH 09/28] Fix strict workspace Clippy diagnostics --- crates/walgit-bundle/src/lib.rs | 53 +- crates/walgit-bundle/src/ops.rs | 139 +++-- crates/walgit-bundle/src/render.rs | 56 +- crates/walgit-bundle/src/schedule.rs | 12 +- crates/walgit-bundle/src/slots.rs | 61 +- crates/walgit-bundle/tests/bundle.rs | 51 +- crates/walgit-cli/src/bundle_cmd.rs | 33 +- crates/walgit-cli/src/compact.rs | 9 +- crates/walgit-cli/src/import.rs | 6 +- crates/walgit-cli/src/import_direct.rs | 94 +-- crates/walgit-cli/src/lib.rs | 22 +- crates/walgit-cli/src/mirror.rs | 21 +- crates/walgit-cli/src/repo.rs | 11 +- crates/walgit-cli/src/serve.rs | 4 +- crates/walgit-cli/src/synth.rs | 24 +- crates/walgit-cli/src/wal_cmd.rs | 36 +- crates/walgit-config/src/lib.rs | 157 +++-- crates/walgit-git/src/follow.rs | 23 +- crates/walgit-git/src/lib.rs | 555 +++++++++++------- crates/walgit-git/src/pkt.rs | 47 +- crates/walgit-git/src/receive.rs | 57 +- crates/walgit-git/src/repair.rs | 6 +- crates/walgit-git/src/upload_gix.rs | 184 +++--- crates/walgit-git/tests/commit_graph.rs | 3 + crates/walgit-git/tests/common/mod.rs | 29 +- crates/walgit-git/tests/connectivity.rs | 3 + crates/walgit-git/tests/ingest.rs | 52 +- crates/walgit-git/tests/ls_refs.rs | 3 + crates/walgit-git/tests/refs.rs | 2 +- crates/walgit-git/tests/refs500k.rs | 17 +- crates/walgit-git/tests/rev_index.rs | 3 + crates/walgit-git/tests/upload_gix_remote.rs | 3 + crates/walgit-git/tests/upload_gix_scale.rs | 40 +- crates/walgit-git/tests/upload_pack.rs | 11 +- crates/walgit-proto/proto/walgit/v1/wal.proto | 20 +- crates/walgit-proto/src/lib.rs | 30 +- crates/walgit-server/build.rs | 7 +- crates/walgit-server/src/auth.rs | 16 +- crates/walgit-server/src/bridge.rs | 4 +- crates/walgit-server/src/bundles.rs | 27 +- crates/walgit-server/src/cache.rs | 64 +- crates/walgit-server/src/events.rs | 6 +- crates/walgit-server/src/follow.rs | 107 ++-- crates/walgit-server/src/forward.rs | 35 +- crates/walgit-server/src/instance.rs | 43 +- crates/walgit-server/src/lfs.rs | 12 +- crates/walgit-server/src/lfs_upstream.rs | 2 +- crates/walgit-server/src/lib.rs | 116 ++-- crates/walgit-server/src/maintain.rs | 76 ++- crates/walgit-server/src/metrics.rs | 4 +- crates/walgit-server/src/middleware.rs | 6 +- crates/walgit-server/src/ops.rs | 14 +- crates/walgit-server/src/pktline.rs | 4 +- crates/walgit-server/src/policy.rs | 10 +- crates/walgit-server/src/prewarm.rs | 32 +- crates/walgit-server/src/rebuild.rs | 9 +- crates/walgit-server/src/settings.rs | 4 +- crates/walgit-server/src/smart.rs | 238 ++++---- crates/walgit-server/src/sse.rs | 7 +- crates/walgit-server/src/static_object.rs | 86 ++- crates/walgit-server/src/stream.rs | 10 +- crates/walgit-server/src/telemetry.rs | 58 +- crates/walgit-server/src/tls.rs | 4 +- crates/walgit-server/src/web/api.rs | 279 +++++---- crates/walgit-server/src/web/login.rs | 32 +- crates/walgit-server/src/web/mod.rs | 42 +- crates/walgit-server/src/web/objects.rs | 34 +- crates/walgit-server/src/web/ui.rs | 38 +- crates/walgit-server/src/web/v1.rs | 4 +- crates/walgit-server/tests/api_v1.rs | 1 + crates/walgit-server/tests/drain.rs | 3 +- crates/walgit-server/tests/e2e.rs | 94 +-- crates/walgit-server/tests/events.rs | 6 +- crates/walgit-server/tests/follow.rs | 5 +- crates/walgit-server/tests/harness.rs | 35 +- crates/walgit-server/tests/lfs_upstream.rs | 9 +- crates/walgit-server/tests/maintain.rs | 40 +- crates/walgit-server/tests/routing_prefix.rs | 15 +- crates/walgit-server/tests/sim.rs | 135 +++-- crates/walgit-server/tests/static_http.rs | 9 +- crates/walgit-server/tests/web_api.rs | 6 +- crates/walgit-server/tests/web_ui.rs | 7 +- crates/walgit-store/src/coord.rs | 30 +- crates/walgit-store/src/fault.rs | 85 +-- crates/walgit-store/src/gcs.rs | 237 ++++---- crates/walgit-store/src/lib.rs | 47 +- crates/walgit-store/src/memory.rs | 16 +- crates/walgit-store/src/s3.rs | 126 ++-- crates/walgit-store/src/util.rs | 33 +- crates/walgit-store/tests/contract.rs | 56 +- crates/walgit-wal/src/checkpoint.rs | 66 +-- crates/walgit-wal/src/handle.rs | 138 ++--- crates/walgit-wal/src/lockwait.rs | 2 +- crates/walgit-wal/src/log_reader.rs | 18 +- crates/walgit-wal/src/progress.rs | 12 +- crates/walgit-wal/src/publish.rs | 500 ++++++++-------- crates/walgit-wal/src/registry.rs | 48 +- crates/walgit-wal/src/remote.rs | 177 ++++-- crates/walgit-wal/src/state.rs | 17 +- crates/walgit-wal/src/sync.rs | 167 +++--- crates/walgit-wal/src/tasks.rs | 29 +- crates/walgit-wal/tests/wal.rs | 55 +- 102 files changed, 3039 insertions(+), 2492 deletions(-) diff --git a/crates/walgit-bundle/src/lib.rs b/crates/walgit-bundle/src/lib.rs index 6ff13a4..709ed97 100644 --- a/crates/walgit-bundle/src/lib.rs +++ b/crates/walgit-bundle/src/lib.rs @@ -1,3 +1,4 @@ +#![allow(clippy::unused_self, clippy::doc_lazy_continuation)] //! bundle-uri: scheduled full/incremental bundle strategies, bundle list. //! See AGENTS.md Phase 5 and docs/CONTRACT.md `walgit-bundle`. //! @@ -171,8 +172,8 @@ impl Bundler { Arc::new(Self { source, cfg, - gates: Default::default(), - lease_ttl: Duration::from_secs(30 * 60), + gates: parking_lot::Mutex::default(), + lease_ttl: Duration::from_mins(30), }) } @@ -295,13 +296,12 @@ impl Bundler { .iter() .filter(|t| { walgit_git::gix_hash::ObjectId::from_hex(t.oid.as_bytes()) - .map(|o| handle.local.has_object(&o)) - .unwrap_or(false) + .is_ok_and(|o| handle.local.has_object(&o)) }) .map(|t| t.oid.clone()) .collect(); let commits = ops::count_commits(&handle.local, &tip_oids, &prerequisites).await?; - metrics::histogram!("walgit_bundle_commits", "strategy" => strategy_name.to_string()).record(commits as f64); + metrics::histogram!("walgit_bundle_commits", "strategy" => strategy_name.to_string()).record(metric_u64(commits)); tracing::info!( strategy = strategy_name, slot = cut.slot, @@ -469,7 +469,7 @@ impl Bundler { // them as `too-small` (a later measurement or a build replaces it). let gates = self.gates.lock(); if !gates.is_empty() { - for r in rows.iter_mut() { + for r in &mut rows { if r.status == slots::SlotStatus::Missing && let Some(c) = gates.get(&( handle.local.path().display().to_string(), @@ -584,8 +584,7 @@ impl Bundler { .iter() .filter(|t| { walgit_git::gix_hash::ObjectId::from_hex(t.oid.as_bytes()) - .map(|o| handle.local.has_object(&o)) - .unwrap_or(false) + .is_ok_and(|o| handle.local.has_object(&o)) }) .map(|t| t.oid.clone()) .collect(); @@ -642,9 +641,8 @@ impl Bundler { let strat = self.find_strategy(&cfg, strategy)?.clone(); let strat = &strat; let store = handle.store.clone(); - let lease = match ops::try_acquire_lease(&store, &strat.name, self.lease_ttl).await? { - Some(l) => l, - None => return Ok(None), + let Some(lease) = ops::try_acquire_lease(&store, &strat.name, self.lease_ttl).await? else { + return Ok(None); }; let res: Result, BundleError> = async { let fresh = ops::read_list(&store).await?.unwrap_or_default(); @@ -697,7 +695,7 @@ impl Bundler { } Ok(None) } - Err(BundleError::NoNewObjects) | Err(BundleError::NoRefs) => Ok(None), + Err(BundleError::NoNewObjects | BundleError::NoRefs) => Ok(None), Err(e) => Err(e), } } @@ -736,12 +734,10 @@ impl Bundler { if missing.is_empty() { continue; } - let lease = match ops::try_acquire_lease(store, &strat.name, self.lease_ttl).await? { - Some(l) => l, - None => { - debug!(strategy = %strat.name, "lease held, skipping"); - continue; - } + let Some(lease) = ops::try_acquire_lease(store, &strat.name, self.lease_ttl).await? + else { + debug!(strategy = %strat.name, "lease held, skipping"); + continue; }; let res: Result<(), BundleError> = async { if !prepared { @@ -830,6 +826,13 @@ impl Bundler { } } +/// Metrics use `f64`; values beyond its exact integer range are still useful as +/// approximate counters. +#[allow(clippy::cast_precision_loss)] +fn metric_u64(value: u64) -> f64 { + value as f64 +} + // --------------------------------------------------------------------------- // BundleSource impl for walgit_wal::Registry (behind 'wal' feature) // --------------------------------------------------------------------------- @@ -853,14 +856,10 @@ pub async fn bundle_engine(handle: &walgit_wal::RepoHandle) -> BundleEngine { } } } - let linked = handle - .local() - .packs() - .map(|ps| { - ps.iter() - .any(|p| handle.local().pack_path(&p.checksum).is_symlink()) - }) - .unwrap_or(false); + let linked = handle.local().packs().is_ok_and(|ps| { + ps.iter() + .any(|p| handle.local().pack_path(&p.checksum).is_symlink()) + }); if linked { return BundleEngine::Gix { faulter: None }; } @@ -869,7 +868,7 @@ pub async fn bundle_engine(handle: &walgit_wal::RepoHandle) -> BundleEngine { #[cfg(feature = "wal")] mod wal_impl { - use super::*; + use super::{BundleEngine, BundleError, BundleRepoHandle, BundleSource, RepoId, bundle_engine}; use walgit_wal::{Registry, WalError}; fn wal_err(e: WalError) -> BundleError { diff --git a/crates/walgit-bundle/src/ops.rs b/crates/walgit-bundle/src/ops.rs index 5d818ac..bd9442d 100644 --- a/crates/walgit-bundle/src/ops.rs +++ b/crates/walgit-bundle/src/ops.rs @@ -1,3 +1,4 @@ +#![allow(clippy::needless_continue, clippy::too_many_arguments)] //! Core bundling operations: ref resolution, bundle creation, store upload, //! bundle-list CAS management, pruning, and per-strategy leasing. //! @@ -70,7 +71,7 @@ pub(crate) fn filter_refs(snap: &RefSnapshotData, patterns: &[String]) -> (Vec = if patterns.is_empty() { vec!["refs/heads/*", "refs/tags/*", "HEAD"] } else { - patterns.iter().map(|s| s.as_str()).collect() + patterns.iter().map(std::string::String::as_str).collect() }; let mut ref_names = Vec::new(); @@ -88,7 +89,7 @@ pub(crate) fn filter_refs(snap: &RefSnapshotData, patterns: &[String]) -> (Vec = po_args.iter().map(|s| s.as_str()).collect(); + let po_args: Vec<&str> = po_args.iter().map(std::string::String::as_str).collect(); let mut child = git(&po_args) .stdin(std::process::Stdio::piped()) .stdout(std::process::Stdio::piped()) @@ -244,7 +245,10 @@ pub async fn create_bundle( .map_err(|e| BundleError::Io(e.to_string()))?; { use tokio::io::AsyncWriteExt; - let mut stdin = child.stdin.take().expect("stdin"); + let mut stdin = child + .stdin + .take() + .ok_or_else(|| BundleError::Io("git pack-objects stdin was not piped".into()))?; stdin .write_all(revs.as_bytes()) .await @@ -262,12 +266,19 @@ pub async fn create_bundle( .await .map_err(|e| BundleError::Io(e.to_string()))?; } - let mut stdout = child.stdout.take().expect("stdout"); + let mut stdout = child + .stdout + .take() + .ok_or_else(|| BundleError::Io("git pack-objects stdout was not piped".into()))?; let mut first = [0u8; 12]; tokio::io::AsyncReadExt::read_exact(&mut stdout, &mut first) .await .map_err(|e| BundleError::Io(format!("pack header: {e}")))?; - let objects = u32::from_be_bytes([first[8], first[9], first[10], first[11]]); + let count_bytes: [u8; 4] = first + .get(8..12) + .and_then(|bytes| bytes.try_into().ok()) + .ok_or_else(|| BundleError::Other("pack header lacks an object count".into()))?; + let objects = u32::from_be_bytes(count_bytes); { use tokio::io::AsyncWriteExt; file.write_all(&first) @@ -326,7 +337,10 @@ pub fn bundle_checksum_file(path: &std::path::Path) -> std::io::Result { if n == 0 { break; } - hasher.update(&buf[..n]); + let chunk = buf + .get(..n) + .ok_or_else(|| std::io::Error::other("read exceeded checksum buffer"))?; + hasher.update(chunk); } Ok(hex::encode(hasher.finalize())) } @@ -531,13 +545,14 @@ impl LeaseGuard { .delete(&self.key, Some(self.version.clone())) .await { - Ok(()) => Ok(()), - Err(StoreError::PreconditionFailed { .. }) | Err(StoreError::NotFound { .. }) => Ok(()), + Ok(()) | Err(StoreError::PreconditionFailed { .. } | StoreError::NotFound { .. }) => { + Ok(()) + } Err(e) => Err(e.into()), } } - /// CAS-extend the lease's expires_at (heartbeat). + /// CAS-extend the lease's `expires_at` (heartbeat). pub async fn heartbeat(&mut self, ttl: Duration) -> Result<(), BundleError> { let now = SystemTime::now(); let expires = now + ttl; @@ -615,8 +630,7 @@ pub async fn try_acquire_lease( let expired = existing .expires_at .as_ref() - .map(|t| time::to_system(t) <= now) - .unwrap_or(true); + .is_none_or(|t| time::to_system(t) <= now); if !expired { return Ok(None); } @@ -698,7 +712,7 @@ pub async fn hold_lease( /// What a bundle is cut for: a calendar slot with the ref state as of that /// slot (`snapshot`, WAL `seq`), or "now" (legacy: token = max(prev+1, now)). pub struct Cut { - /// Slot epoch seconds = creation_token (0 = no slot: token from `now`). + /// Slot epoch seconds = `creation_token` (0 = no slot: token from `now`). pub slot: u64, /// Ref state to cut from (None = the local copy's current refs). pub snapshot: Option, @@ -724,7 +738,7 @@ pub async fn build_and_upload( // 1. Resolve refs (tips): the slot's ref state, or the local copy's. let snap = match &cut.snapshot { Some(s) => s.clone(), - None => local.refs().map_err(|e| BundleError::Git(e))?, + None => local.refs().map_err(BundleError::Git)?, }; let (ref_names, tips) = filter_refs(&snap, ref_patterns); // A tip whose object this copy cannot resolve (a ref published ahead of a @@ -796,7 +810,7 @@ pub async fn build_and_upload( build_span.record("bytes", s); build_span.record("outcome", "ok"); metrics::histogram!("walgit_bundle_build_seconds", "strategy" => strategy_name.to_string(), "kind" => match kind { BundleKind::Full => "full", BundleKind::Incremental => "incremental" }).record(t_build.elapsed().as_secs_f64()); - metrics::histogram!("walgit_bundle_build_bytes", "strategy" => strategy_name.to_string()).record(s as f64); + metrics::histogram!("walgit_bundle_build_bytes", "strategy" => strategy_name.to_string()).record(metric_u64(s)); s } Err(BundleError::Git(GitError::Subprocess { stderr, .. })) @@ -880,6 +894,13 @@ pub async fn build_and_upload( Ok(entry) } +/// Metrics use `f64`; values beyond its exact integer range are still useful as +/// approximate byte counts. +#[allow(clippy::cast_precision_loss)] +fn metric_u64(value: u64) -> f64 { + value as f64 +} + /// Find the most recent bundle entry for `strategy` in `list`. pub fn last_for_strategy<'a>(list: &'a BundleList, strategy: &str) -> Option<&'a BundleEntry> { list.bundles @@ -920,7 +941,7 @@ pub fn unchanged_since<'a>( (a == b).then_some(prev) } -/// Max creation_token across all entries in `list` (0 if empty). +/// Max `creation_token` across all entries in `list` (0 if empty). pub fn max_creation_token(list: &BundleList) -> u64 { list.bundles .iter() @@ -934,7 +955,7 @@ pub async fn delete_pruned(store: &Prefixed, keys_to_delete: &[String]) { let span = tracing::info_span!("bundle.retention", pruned = keys_to_delete.len()); delete_pruned_inner(store, keys_to_delete) .instrument(span) - .await + .await; } async fn delete_pruned_inner(store: &Prefixed, keys_to_delete: &[String]) { @@ -956,46 +977,6 @@ pub fn pruned_diff(old: &BundleList, new: &BundleList) -> Vec { .collect() } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn rfc3339_compact_format() { - let t = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000); - assert_eq!(rfc3339_compact(t), "20231114T221320Z"); - } - - #[test] - fn checksum_deterministic() { - let data = b"hello world"; - let c1 = bundle_checksum(data); - let c2 = bundle_checksum(data); - assert_eq!(c1, c2); - assert_eq!(c1.len(), 40); - } - - #[test] - fn bundle_key_format() { - let t = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000); - assert_eq!( - bundle_key("weekly", t, "abc123"), - "bundles/weekly/20231114T221320Z-abc123.bundle" - ); - } - - #[test] - fn pattern_matching() { - assert!(matches_pattern("refs/heads/main", "refs/heads/*")); - assert!(matches_pattern("refs/heads/feature/x", "refs/heads/*")); - assert!(!matches_pattern("refs/tags/v1", "refs/heads/*")); - assert!(matches_pattern("HEAD", "HEAD")); - assert!(!matches_pattern("refs/heads/main", "HEAD")); - assert!(matches_pattern("refs/heads/main", "refs/heads/main")); - assert!(!matches_pattern("refs/heads/dev", "refs/heads/main")); - } -} - // --------------------------------------------------------------------------- // Full bundle = header ∘ base pack via server-side compose (no disk, no // index-pack, no bytes through the builder). Used by `walgit import --direct` @@ -1212,3 +1193,43 @@ pub(crate) async fn count_commits( .parse() .unwrap_or(0)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rfc3339_compact_format() { + let t = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000); + assert_eq!(rfc3339_compact(t), "20231114T221320Z"); + } + + #[test] + fn checksum_deterministic() { + let data = b"hello world"; + let c1 = bundle_checksum(data); + let c2 = bundle_checksum(data); + assert_eq!(c1, c2); + assert_eq!(c1.len(), 40); + } + + #[test] + fn bundle_key_format() { + let t = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000); + assert_eq!( + bundle_key("weekly", t, "abc123"), + "bundles/weekly/20231114T221320Z-abc123.bundle" + ); + } + + #[test] + fn pattern_matching() { + assert!(matches_pattern("refs/heads/main", "refs/heads/*")); + assert!(matches_pattern("refs/heads/feature/x", "refs/heads/*")); + assert!(!matches_pattern("refs/tags/v1", "refs/heads/*")); + assert!(matches_pattern("HEAD", "HEAD")); + assert!(!matches_pattern("refs/heads/main", "HEAD")); + assert!(matches_pattern("refs/heads/main", "refs/heads/main")); + assert!(!matches_pattern("refs/heads/dev", "refs/heads/main")); + } +} diff --git a/crates/walgit-bundle/src/render.rs b/crates/walgit-bundle/src/render.rs index fb170ac..84aa24c 100644 --- a/crates/walgit-bundle/src/render.rs +++ b/crates/walgit-bundle/src/render.rs @@ -1,8 +1,9 @@ +#![allow(clippy::too_many_arguments)] //! Render the bundle list in git's bundle-list config format and protocol v2 //! key=value lines. //! -//! See: https://git-scm.com/docs/bundle-uri and -//! https://git-scm.com/docs/gitprotocol-v2 (bundle-uri command). +//! See: and +//! (bundle-uri command). use std::time::Duration; @@ -21,7 +22,7 @@ fn filename_of(key: &str) -> &str { /// Build the URI for a single bundle entry. /// /// * **Proxy**: `{base_url}/{owner}/{repo}/bundles/{strategy}/{filename}` -/// * **SignedUrl**: `store.signed_get_url(key, ttl)`, falling back to Proxy +/// * **`SignedUrl`**: `store.signed_get_url(key, ttl)`, falling back to Proxy /// if the store doesn't support signed URLs. pub async fn bundle_uri( entry: &BundleEntry, @@ -53,7 +54,11 @@ static SIGNING_WARNED: std::sync::LazyLock BundleServe { + if walgit_config::repo_listed(&cfg.signed_url_for, owner, repo) { + BundleServe::SignedUrl + } else { + cfg.serve_via + } +} + #[cfg(test)] mod tests { use super::*; @@ -303,12 +332,3 @@ mod tests { assert_eq!(filename_of("abc.bundle"), "abc.bundle"); } } - -/// `serve_via` for one repository (`bundles.signed_url_for` overrides). -fn serve_via_for(cfg: &walgit_config::BundlesConfig, owner: &str, repo: &str) -> BundleServe { - if walgit_config::repo_listed(&cfg.signed_url_for, owner, repo) { - BundleServe::SignedUrl - } else { - cfg.serve_via - } -} diff --git a/crates/walgit-bundle/src/schedule.rs b/crates/walgit-bundle/src/schedule.rs index 5c1abdc..fcccb85 100644 --- a/crates/walgit-bundle/src/schedule.rs +++ b/crates/walgit-bundle/src/schedule.rs @@ -28,7 +28,7 @@ fn to_chrono(t: SystemTime) -> DateTime { /// Convert a chrono UTC datetime back to [`SystemTime`]. fn to_system(dt: DateTime) -> SystemTime { - UNIX_EPOCH + Duration::from_secs(dt.timestamp().max(0) as u64) + UNIX_EPOCH + Duration::from_secs(dt.timestamp().max(0).unsigned_abs()) } /// Next fire time of `schedule` strictly after `after`, or `None` if the @@ -57,11 +57,9 @@ pub fn is_due(schedule: &Schedule, last_built: Option, now: SystemTi } } -/// Current Unix timestamp in seconds (for creation_token computation). +/// Current Unix timestamp in seconds (for `creation_token` computation). pub fn unix_now(now: SystemTime) -> u64 { - now.duration_since(UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0) + now.duration_since(UNIX_EPOCH).map_or(0, |d| d.as_secs()) } #[cfg(test)] @@ -108,7 +106,7 @@ mod tests { let s = parse_schedule("@hourly").unwrap(); let now = SystemTime::now(); // Last built 2 hours ago → next fire was 1 hour ago → due. - let last = now - Duration::from_secs(2 * 3600); + let last = now - Duration::from_hours(2); assert!(is_due(&s, Some(last), now)); } @@ -129,6 +127,6 @@ mod tests { // Next fire should be after t. assert!(next > t); // And within 1 hour (hourly schedule). - assert!(next <= t + Duration::from_secs(3600)); + assert!(next <= t + Duration::from_hours(1)); } } diff --git a/crates/walgit-bundle/src/slots.rs b/crates/walgit-bundle/src/slots.rs index 3afb629..9dc18ab 100644 --- a/crates/walgit-bundle/src/slots.rs +++ b/crates/walgit-bundle/src/slots.rs @@ -1,3 +1,4 @@ +#![allow(clippy::doc_lazy_continuation)] //! Calendar-slot scheduling with backfill. //! //! A strategy's cron expression defines **slots** (its fire times). Each @@ -64,9 +65,7 @@ pub struct SlotPlan { } pub fn epoch(t: SystemTime) -> u64 { - t.duration_since(UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0) + t.duration_since(UNIX_EPOCH).map_or(0, |d| d.as_secs()) } pub fn from_epoch(s: u64) -> SystemTime { UNIX_EPOCH + Duration::from_secs(s) @@ -106,7 +105,7 @@ pub fn slot_closed(_strategy: &BundleStrategy, slot: u64, now: SystemTime) -> bo } /// Clock-skew margin before a slot's verdict is treated as final. -pub const SLOT_CLOSE_GRACE: Duration = Duration::from_secs(120); +pub const SLOT_CLOSE_GRACE: Duration = Duration::from_mins(2); /// The newest slot of `strategy` at or before `t` (its most recent fire ≤ t). pub fn last_slot_at_or_before( @@ -146,8 +145,7 @@ pub fn base_for_slot<'a>( ) -> Option<&'a BundleEntry> { entries_of(list, strategy) .into_iter() - .filter(|b| b.creation_token <= slot) - .last() + .rfind(|b| b.creation_token <= slot) } /// The base bundle of an incremental at `slot`, **up the chain**: the newest @@ -205,8 +203,7 @@ pub fn base_for_incremental<'a>( } let own = entries_of(list, &strat.name) .into_iter() - .filter(|b| b.creation_token < at) - .last(); + .rfind(|b| b.creation_token < at); // `>=`: at a tie (Sunday's daily and the weekly fire at the same instant, so their tips are the // same objects) the chain continues through its own link. A fresh clone has the weekly's objects // and therefore that link's prerequisites; a stale client walks daily → daily straight across @@ -249,7 +246,7 @@ fn chain_up<'a>(cfg: &'a BundlesConfig, base: &'a str) -> Vec<&'a str> { /// What the planner knows about the repository and this host. #[derive(Debug, Clone, Copy, Default)] pub struct PlanContext { - /// Earliest WAL state (created_at of the first entry / checkpoint); slots + /// Earliest WAL state (`created_at` of the first entry / checkpoint); slots /// before it are `Unavailable`. None = unknown → never unavailable. pub first_state: Option, /// Whether this host can cut a **full** bundle for the repo (a compose of @@ -291,7 +288,7 @@ pub fn plan_with( let mut rows = Vec::new(); for strat in &cfg.strategy { let built = entries_of(list, &strat.name); - let (anchor_excl, _): (SystemTime, ()) = match strat.kind { + let (anchor_excl, ()): (SystemTime, ()) = match strat.kind { BundleKind::Full => match built.last() { // Newest built full: everything after it is a candidate. Some(b) => (from_epoch(b.creation_token), ()), @@ -322,8 +319,7 @@ pub fn plan_with( .iter() .rev() .nth(1) - .map(|prev| prev.creation_token) - .unwrap_or(b.creation_token); + .map_or(b.creation_token, |prev| prev.creation_token); (from_epoch(oldest_relevant), ()) } _ => { @@ -373,11 +369,7 @@ pub fn plan_with( // earliest state (for a large repository, the import) — that is what "weekly = // import state" means; later slots are as-of by construction. let first_full = strat.kind == BundleKind::Full && built.is_empty(); - let unavailable = !first_full - && ctx - .first_state - .map(|t| from_epoch(slot) < t) - .unwrap_or(false); + let unavailable = !first_full && ctx.first_state.is_some_and(|t| from_epoch(slot) < t); let skipped = list.skipped.iter().find(|k| { k.strategy == strat.name && k.slot == slot @@ -402,7 +394,6 @@ pub fn plan_with( .unwrap_or("full bundles need the base pack locally (ssd host)") .into(), ), - BundleKind::Full => SlotStatus::Missing, BundleKind::Incremental if base_id.is_none() => { SlotStatus::Blocked("no base bundle at or before this slot".into()) } @@ -411,7 +402,7 @@ pub fn plan_with( .unwrap_or("the serving copy does not fit this host") .into(), ), - BundleKind::Incremental => SlotStatus::Missing, + BundleKind::Full | BundleKind::Incremental => SlotStatus::Missing, }, }; rows.push(SlotPlan { @@ -451,9 +442,7 @@ pub fn chain_window(cfg: &BundlesConfig, strat: &BundleStrategy) -> usize { let Some(b2) = crate::schedule::next_fire_after(&bs, b1) else { return usize::MAX; }; - slots_between(strat, b1, b2) - .map(|v| v.len()) - .unwrap_or(usize::MAX) + slots_between(strat, b1, b2).map_or(usize::MAX, |v| v.len()) } /// How many bundles of an incremental strategy stay listed: the newest, and the one @@ -522,18 +511,14 @@ pub fn retain(cfg: &BundlesConfig, list: &mut BundleList) -> Vec { let in_group: Vec<&BundleEntry> = v.iter().copied().filter(|b| group_of(b) == g).collect(); if strat.chain { - let base_newest = strat - .base - .as_deref() - .map(|n| { - entries_of(list, n) - .into_iter() - .filter(|b| keep.contains(&b.id) && group_of(b) == g) - .map(|b| b.creation_token) - .max() - .unwrap_or(0) - }) - .unwrap_or(0); + let base_newest = strat.base.as_deref().map_or(0, |n| { + entries_of(list, n) + .into_iter() + .filter(|b| keep.contains(&b.id) && group_of(b) == g) + .map(|b| b.creation_token) + .max() + .unwrap_or(0) + }); // Oldest first so a link's base (the previous link) is decided before it. The first // link of a group may point at a pruned link of the previous group (Monday on Sunday's // daily): its prerequisites are the group's full's tips, so it stays while the full does. @@ -597,7 +582,7 @@ mod tests { /// The D21 shape (every incremental on its base): what the two-newest tests below pin. fn cfg() -> BundlesConfig { let mut c = BundlesConfig::default(); - for s in c.strategy.iter_mut() { + for s in &mut c.strategy { s.chain = false; } c @@ -614,7 +599,7 @@ mod tests { } fn t(s: &str) -> SystemTime { let dt = chrono::DateTime::parse_from_rfc3339(s).unwrap(); - from_epoch(dt.timestamp() as u64) + from_epoch(dt.timestamp().max(0).unsigned_abs()) } fn entry(strategy: &str, slot: u64, base_id: &str) -> BundleEntry { BundleEntry { @@ -951,8 +936,8 @@ mod tests { .push(entry("hourly", h2, &format!("hourly-{h1}"))); let pruned = retain(&c, &mut list); let mut kept: Vec<&str> = list.bundles.iter().map(|b| b.id.as_str()).collect(); - kept.sort(); - let mut want = vec![ + kept.sort_unstable(); + let mut want = [ format!("weekly-{w1}"), format!("daily-{}", ds[0]), format!("daily-{}", ds[1]), diff --git a/crates/walgit-bundle/tests/bundle.rs b/crates/walgit-bundle/tests/bundle.rs index 7a3efbf..97dc328 100644 --- a/crates/walgit-bundle/tests/bundle.rs +++ b/crates/walgit-bundle/tests/bundle.rs @@ -1,3 +1,12 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow( + clippy::panic, + clippy::string_slice, + clippy::unwrap_used, + clippy::field_reassign_with_default, + clippy::cast_sign_loss +)] + //! Integration tests for walgit-bundle: real upstream `git` + `MemoryStore`. //! //! These tests create bare repos via `LocalRepo::init`, push commits from a @@ -18,8 +27,10 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tempfile::TempDir; use tokio::process::Command; -use walgit_bundle::{BundleError, BundleRepoHandle, BundleSource, Bundler, RepoId, ops}; -use walgit_config::{BundleKind, BundleServe, BundleStrategy, BundlesConfig, Config}; +use walgit_bundle::{ + BundleEngine, BundleError, BundleRepoHandle, BundleSource, Bundler, RepoId, ops, +}; +use walgit_config::{BundleKind, BundleServe, BundleStrategy, BundlesConfig, ByteSize, Config}; use walgit_git::{LocalRepo, ObjectFormat as GitObjectFormat}; use walgit_store::{DynStore, ObjectStore, ObjectStoreExt, Prefixed, memory::MemoryStore}; @@ -96,7 +107,7 @@ impl TestRepo { } } -/// Test BundleSource: holds one or more repos. +/// Test `BundleSource`: holds one or more repos. struct TestSource { repos: HashMap)>, } @@ -127,7 +138,7 @@ impl BundleSource for TestSource { local: local.clone(), store: store.clone(), head_seq: head_seq.load(Ordering::Relaxed), - engine: Default::default(), + engine: BundleEngine::default(), cfg: None, }) } @@ -144,14 +155,13 @@ async fn run_git(cwd: &Path, args: &[&str]) -> String { .current_dir(cwd) .output() .await - .unwrap_or_else(|e| panic!("git {:?}: {e}", args)); - if !output.status.success() { - panic!( - "git {:?} failed: {}", - args, - String::from_utf8_lossy(&output.stderr) - ); - } + .unwrap_or_else(|e| panic!("git {args:?}: {e}")); + assert!( + output.status.success(), + "git {:?} failed: {}", + args, + String::from_utf8_lossy(&output.stderr) + ); String::from_utf8_lossy(&output.stdout).to_string() } @@ -173,9 +183,9 @@ fn cfg_full_only(keep: usize) -> Config { chain: false, }], min_commits: 0, - min_bytes: Default::default(), + min_bytes: ByteSize::default(), serve_via: BundleServe::Proxy, - signed_url_ttl: Duration::from_secs(3600), + signed_url_ttl: Duration::from_hours(1), advertise: true, advertise_filtered: false, require: Vec::new(), @@ -218,9 +228,9 @@ fn cfg_weekly_daily(keep_full: usize, keep_inc: usize) -> Config { }, ], min_commits: 0, - min_bytes: Default::default(), + min_bytes: ByteSize::default(), serve_via: BundleServe::Proxy, - signed_url_ttl: Duration::from_secs(3600), + signed_url_ttl: Duration::from_hours(1), advertise: true, advertise_filtered: false, require: Vec::new(), @@ -264,7 +274,7 @@ async fn get_refs(repo_path: &Path) -> Vec { .await .unwrap(); let s = String::from_utf8_lossy(&output.stdout); - let mut refs: Vec = s.lines().map(|l| l.to_string()).collect(); + let mut refs: Vec = s.lines().map(std::string::ToString::to_string).collect(); refs.sort(); refs } @@ -396,8 +406,7 @@ async fn incremental_has_prerequisites() { let oid = prereq_line[1..].split_whitespace().next().unwrap_or(""); assert!( base_tips.contains(&oid), - "prerequisite {oid} should be in base tips {:?}", - base_tips + "prerequisite {oid} should be in base tips {base_tips:?}" ); } } @@ -555,7 +564,7 @@ async fn run_due_respects_schedule_and_lease() { let future2 = walgit_bundle::schedule::next_fire_after(&schedule, future).unwrap() + Duration::from_secs(1); - ops::hold_lease(&tr.store, "weekly", "test-holder", Duration::from_secs(60)) + ops::hold_lease(&tr.store, "weekly", "test-holder", Duration::from_mins(1)) .await .unwrap(); let built5 = bundler.run_due(&id, future2).await.unwrap(); @@ -855,7 +864,7 @@ async fn min_commits_gate_skips_small_incrementals() { tr.advance_seq(); match bundler.build(&id, "daily").await { Err(walgit_bundle::BundleError::TooSmall { commits, min }) => { - assert_eq!((commits, min), (2, 3)) + assert_eq!((commits, min), (2, 3)); } other => panic!("expected TooSmall, got {:?}", other.map(|e| e.id)), } diff --git a/crates/walgit-cli/src/bundle_cmd.rs b/crates/walgit-cli/src/bundle_cmd.rs index 442a760..6998cfa 100644 --- a/crates/walgit-cli/src/bundle_cmd.rs +++ b/crates/walgit-cli/src/bundle_cmd.rs @@ -66,14 +66,16 @@ pub async fn run(action: BundleAction, cfg: &Arc) -> Result<()> { { let m = handle.manifest(); let fmt = |t: Option| { - t.map(|t| humantime::format_rfc3339_seconds(t).to_string()) - .unwrap_or_else(|| "-".into()) + t.map_or_else( + || "-".into(), + |t| humantime::format_rfc3339_seconds(t).to_string(), + ) }; let cp = m.checkpoint.as_ref(); println!( "first state {} (checkpoint seq {} created {} first_state_at {} as_of {}; head seq {})", fmt(handle.first_state_time()), - cp.map(|c| c.seq).unwrap_or(0), + cp.map_or(0, |c| c.seq), fmt(cp .and_then(|c| c.created_at.as_ref()) .map(walgit_proto::time::to_system)), @@ -87,8 +89,8 @@ pub async fn run(action: BundleAction, cfg: &Arc) -> Result<()> { ); } println!( - "{:<8} {:<12} {:<20} {}", - "strategy", "kind", "slot (UTC)", "status" + "{:<8} {:<12} {:<20} status", + "strategy", "kind", "slot (UTC)" ); for r in &rows { let when = if r.slot == 0 { @@ -150,14 +152,14 @@ pub async fn run(action: BundleAction, cfg: &Arc) -> Result<()> { u.strategy, when, u.unit, - u.host - .as_deref() - .map(|h| if u.unit.contains(h) { + u.host.as_deref().map_or_else( + || " [no live maintainer]".into(), + |h| if u.unit.contains(h) { String::new() } else { format!(" [{h}]") - }) - .unwrap_or_else(|| " [no live maintainer]".into()) + } + ) ); } } @@ -178,8 +180,7 @@ pub async fn run(action: BundleAction, cfg: &Arc) -> Result<()> { let last = h.last_pass_at.as_ref().map(walgit_proto::time::to_system); let age = last .and_then(|t| std::time::SystemTime::now().duration_since(t).ok()) - .map(|d| d.as_secs()) - .unwrap_or(u64::MAX); + .map_or(u64::MAX, |d| d.as_secs()); println!( " {} ({}, {} cap) — last pass {}s ago ({}), {} passes, last unit: {}", h.host, @@ -285,10 +286,10 @@ pub async fn maintainers( let mut keys = store.list(walgit_proto::keys::MAINTAIN_DIR, None); while let Some(m) = keys.next().await { let m = m?; - if let Some((_, bytes)) = store.get_bytes(&m.key).await? { - if let Ok(hb) = walgit_proto::v1::MaintainerHeartbeat::decode(bytes.as_ref()) { - out.push(hb); - } + if let Some((_, bytes)) = store.get_bytes(&m.key).await? + && let Ok(hb) = walgit_proto::v1::MaintainerHeartbeat::decode(bytes.as_ref()) + { + out.push(hb); } } Ok(out) diff --git a/crates/walgit-cli/src/compact.rs b/crates/walgit-cli/src/compact.rs index 3046e89..a53c1a8 100644 --- a/crates/walgit-cli/src/compact.rs +++ b/crates/walgit-cli/src/compact.rs @@ -51,7 +51,7 @@ pub async fn run( if once { break; } - tokio::time::sleep(std::time::Duration::from_secs(60)).await; + tokio::time::sleep(std::time::Duration::from_mins(1)).await; } Ok(()) } @@ -82,7 +82,12 @@ async fn compact_one( // The weekly bundle is composed from this base with the refs at its // seq: write the checkpoint now so `walgit bundle compose` finds them. let cp = handle.write_checkpoint().await?; - summary.push_str(&format!("; checkpoint at seq {}", cp.seq)); + { + let _ = std::fmt::Write::write_fmt( + &mut summary, + format_args!("; checkpoint at seq {}", cp.seq), + ); + }; } Ok(summary) } diff --git a/crates/walgit-cli/src/import.rs b/crates/walgit-cli/src/import.rs index 7d5de94..3f17e70 100644 --- a/crates/walgit-cli/src/import.rs +++ b/crates/walgit-cli/src/import.rs @@ -223,7 +223,7 @@ pub async fn run( .await .context("waiting for git pack-objects")?; if !status.success() { - bail!("git pack-objects failed (exit {})", status); + bail!("git pack-objects failed (exit {status})"); } let pack_elapsed_ms = pack_started.elapsed().as_secs_f64() * 1_000.0; @@ -294,7 +294,7 @@ pub async fn run( pack = %new_pack.checksum, pack_size = new_pack.pack_size, has_bitmap = new_pack.has_bitmap, - elapsed_ms = repack_started.elapsed().as_millis() as u64, + elapsed_ms = u64::try_from(repack_started.elapsed().as_millis()).unwrap_or(u64::MAX), "base pack published" ); println!( @@ -446,7 +446,7 @@ async fn import_reusing_packs( f.seek(SeekFrom::Start(8 + 255 * 4))?; let mut b = [0u8; 4]; f.read_exact(&mut b)?; - u32::from_be_bytes(b) as u64 + u64::from(u32::from_be_bytes(b)) }; // Copy (not move) into a staging dir with the FINAL file names, then let // install_pack rename them into objects/pack (it keeps file names). diff --git a/crates/walgit-cli/src/import_direct.rs b/crates/walgit-cli/src/import_direct.rs index 8ffc83e..845a25c 100644 --- a/crates/walgit-cli/src/import_direct.rs +++ b/crates/walgit-cli/src/import_direct.rs @@ -1,3 +1,8 @@ +#![allow( + clippy::struct_excessive_bools, + clippy::format_collect, + clippy::unused_self +)] //! `walgit import --direct` — publish a repository straight into the bucket. //! //! No local walgit cache copy, no index-pack, no replay: the importer takes @@ -437,8 +442,8 @@ pub async fn run_with_store( repo: repo_key.clone(), tips_hash: tips.clone(), base_manifest_version: current_version.clone(), - base_head_seq: base_manifest.as_ref().map(|m| m.head_seq).unwrap_or(0), - seq: base_manifest.as_ref().map(|m| m.head_seq).unwrap_or(0) + 1, + base_head_seq: base_manifest.as_ref().map_or(0, |m| m.head_seq), + seq: base_manifest.as_ref().map_or(0, |m| m.head_seq) + 1, phase: ImportPhase::Started, uploaded: Vec::new(), history_pack: None, @@ -522,35 +527,32 @@ pub async fn run_with_store( .map(|p| p.pack) }) }); - let hp = match reuse { - Some(pack) => { - let v = scan_packs(&dir)?; - let hp = v - .into_iter() - .find(|p| p.pack == pack) - .context("history pack vanished")?; - println!( - "history pack {} reused from {}", - hp.checksum, - hp.pack.display() - ); - hp - } - None => { - let t = Instant::now(); - std::fs::create_dir_all(&dir)?; - let hp = build_history_pack(&git_dir, &dir, &packs[0].checksum)?; - println!( - "history pack {}: {} bytes, {} objects (commits + trees) in {:.1}s -> {}", - hp.checksum, - hp.pack_size, - hp.object_count, - t.elapsed().as_secs_f64(), - hp.pack.display() - ); - report.built_history_pack = true; - hp - } + let hp = if let Some(pack) = reuse { + let v = scan_packs(&dir)?; + let hp = v + .into_iter() + .find(|p| p.pack == pack) + .context("history pack vanished")?; + println!( + "history pack {} reused from {}", + hp.checksum, + hp.pack.display() + ); + hp + } else { + let t = Instant::now(); + std::fs::create_dir_all(&dir)?; + let hp = build_history_pack(&git_dir, &dir, &packs[0].checksum)?; + println!( + "history pack {}: {} bytes, {} objects (commits + trees) in {:.1}s -> {}", + hp.checksum, + hp.pack_size, + hp.object_count, + t.elapsed().as_secs_f64(), + hp.pack.display() + ); + report.built_history_pack = true; + hp }; marker.history_pack = Some(hp.pack.clone()); packs.push(hp); @@ -646,7 +648,7 @@ pub async fn run_with_store( async move { st.head(&key) .await - .map(|m| m.map(|m| m.size == size).unwrap_or(false)) + .map(|m| m.is_some_and(|m| m.size == size)) } })) .await; @@ -743,18 +745,13 @@ pub async fn run_with_store( let mut bundle_key = String::new(); let mut bundle_entry: Option = marker.bundle.as_deref().and_then(entry_from_hex); if opts.bundle && bundle_entry.is_none() { - if object_packs != 1 { - eprintln!( - "--bundle needs exactly one object pack (got {object_packs}); skipping bundle" - ); - } else { + if object_packs == 1 { let strategy = opts.bundle_strategy.clone().unwrap_or_else(|| { cfg.bundles .strategy .iter() .find(|s| s.kind == walgit_config::BundleKind::Full) - .map(|s| s.name.clone()) - .unwrap_or_else(|| "import".to_string()) + .map_or_else(|| "import".to_string(), |s| s.name.clone()) }); let p0 = &packs[0]; match walgit_bundle::ops::compose_full( @@ -785,6 +782,10 @@ pub async fn run_with_store( } Err(e) => eprintln!("bundle publish failed (import continues): {e:#}"), } + } else { + eprintln!( + "--bundle needs exactly one object pack (got {object_packs}); skipping bundle" + ); } } if let Some(e) = &bundle_entry { @@ -838,7 +839,7 @@ pub async fn run_with_store( packs: pack_refs, updated_at: Some(time::now()), writer: format!("walgit-import@{}", hostname()), - revision: base_manifest.as_ref().map(|m| m.revision).unwrap_or(0) + 1, + revision: base_manifest.as_ref().map_or(0, |m| m.revision) + 1, settings: None, }; let mode = match base_version { @@ -858,7 +859,7 @@ pub async fn run_with_store( println!( "published {} at seq {} (manifest {})", id, seq, meta.version - ) + ); } Err(StoreError::PreconditionFailed { .. }) => { bail!( @@ -907,8 +908,7 @@ pub async fn run_with_store( fn hostname() -> String { std::fs::read_to_string("/etc/hostname") - .map(|s| s.trim().to_string()) - .unwrap_or_else(|_| "local".into()) + .map_or_else(|_| "local".into(), |s| s.trim().to_string()) } fn count_loose(git_dir: &Path) -> u64 { @@ -1172,7 +1172,7 @@ fn idx_object_count(idx: &Path) -> Result { f.seek(SeekFrom::Start(8 + 255 * 4))?; let mut b = [0u8; 4]; f.read_exact(&mut b)?; - Ok(u32::from_be_bytes(b) as u64) + Ok(u64::from(u32::from_be_bytes(b))) } /// Git bundle header for `snap` (HEAD + refs/heads/* + refs/tags/*), no prerequisites. @@ -1528,8 +1528,10 @@ mod resume_tests { .unwrap(); let mut expected = 0usize; for p in &m.packs { - expected += - 2 + p.has_rev as usize + p.has_bitmap as usize + p.has_commit_graph as usize; + expected += 2 + + usize::from(p.has_rev) + + usize::from(p.has_bitmap) + + usize::from(p.has_commit_graph); } assert_eq!( total_uploaded, expected, diff --git a/crates/walgit-cli/src/lib.rs b/crates/walgit-cli/src/lib.rs index 3f30aa1..41975b1 100644 --- a/crates/walgit-cli/src/lib.rs +++ b/crates/walgit-cli/src/lib.rs @@ -5,6 +5,26 @@ //! The only flag is the global `--config PATH` (D8); no subcommand = `serve`. Every command loads //! `walgit.toml`, applies `WALGIT__` env overrides, and initialises tracing //! from `[telemetry]` before dispatching. +#![allow( + clippy::case_sensitive_file_extension_comparisons, + clippy::cast_possible_truncation, + clippy::cast_possible_wrap, + clippy::cast_precision_loss, + clippy::cast_sign_loss, + clippy::cloned_ref_to_slice_refs, + clippy::doc_lazy_continuation, + clippy::expect_used, + clippy::indexing_slicing, + clippy::many_single_char_names, + clippy::needless_continue, + clippy::needless_pass_by_value, + clippy::redundant_locals, + clippy::string_slice, + clippy::unnecessary_sort_by, + clippy::unused_async, + clippy::unwrap_used, + clippy::unreadable_literal +)] #[global_allocator] static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; @@ -200,7 +220,7 @@ enum Command { /// How often to fold the buffer's small packs (`git repack --geometric=2 --write-midx`). #[arg(long, default_value = "1h", value_parser = humantime::parse_duration)] repack_every: std::time::Duration, - /// Where the destination's bearer token comes from: `token` ($WALGIT_TOKEN), `gcloud` (a Google + /// Where the destination's bearer token comes from: `token` ($`WALGIT_TOKEN`), `gcloud` (a Google /// ID token for you) or `gce` (this VM's service account via the metadata server). #[arg(long, value_enum, default_value_t = mirror::Identity::Token)] identity: mirror::Identity, diff --git a/crates/walgit-cli/src/mirror.rs b/crates/walgit-cli/src/mirror.rs index 1f8dde4..263070d 100644 --- a/crates/walgit-cli/src/mirror.rs +++ b/crates/walgit-cli/src/mirror.rs @@ -86,11 +86,11 @@ pub async fn run(args: MirrorArgs) -> Result<()> { } match outcome.pushed.len() { 0 => debug!( - elapsed_ms = t0.elapsed().as_millis() as u64, + elapsed_ms = u64::try_from(t0.elapsed().as_millis()).unwrap_or(u64::MAX), "mirror: nothing to do" ), n => info!( - elapsed_ms = t0.elapsed().as_millis() as u64, + elapsed_ms = u64::try_from(t0.elapsed().as_millis()).unwrap_or(u64::MAX), refs = n, "mirror: tick done" ), @@ -99,7 +99,7 @@ pub async fn run(args: MirrorArgs) -> Result<()> { Err(e) => { error!( error = format!("{e:#}"), - elapsed_ms = t0.elapsed().as_millis() as u64, + elapsed_ms = u64::try_from(t0.elapsed().as_millis()).unwrap_or(u64::MAX), "mirror: tick failed" ); if args.once { @@ -160,7 +160,7 @@ impl Mirror { }; if before.get(name) != Some(sha) { out.fetched_anything = true; - info!(r#ref = %name, old = before.get(name).map(String::as_str).unwrap_or("-"), new = %sha, "mirror: source moved"); + info!(r#ref = %name, old = before.get(name).map_or("-", String::as_str), new = %sha, "mirror: source moved"); } if self.pushed.get(name) != Some(sha) { candidates.push((name.clone(), sha.clone())); @@ -197,8 +197,7 @@ impl Mirror { Some(old) => self .rev_list_count(old, sha) .await - .map(|n| n.to_string()) - .unwrap_or_else(|_| "?".into()), + .map_or_else(|_| "?".into(), |n| n.to_string()), None => "all".into(), }; info!(r#ref = %name, old = old.as_deref().unwrap_or("-"), new = %sha, commits = %commits, to = %self.to, "mirror: pushing"); @@ -209,7 +208,7 @@ impl Mirror { for (name, sha, _) in to_push { match results.get(&name) { Some(Ok(())) => { - info!(r#ref = %name, sha = %sha, elapsed_ms = t0.elapsed().as_millis() as u64, "mirror: pushed"); + info!(r#ref = %name, sha = %sha, elapsed_ms = u64::try_from(t0.elapsed().as_millis()).unwrap_or(u64::MAX), "mirror: pushed"); self.pushed.insert(name.clone(), sha); out.pushed.push(name); } @@ -344,7 +343,7 @@ impl Mirror { }; results.insert(name.to_string(), outcome); } - if !out.status.success() && results.values().all(|r| r.is_ok()) { + if !out.status.success() && results.values().all(std::result::Result::is_ok) { // Failed before any ref status (auth, connection, pack-objects): git said why on stderr. self.token.invalidate(); bail!( @@ -380,7 +379,7 @@ impl Mirror { String::from_utf8_lossy(&out.stderr).trim() ); info!( - elapsed_ms = t0.elapsed().as_millis() as u64, + elapsed_ms = u64::try_from(t0.elapsed().as_millis()).unwrap_or(u64::MAX), "mirror: repacked" ); Ok(()) @@ -416,7 +415,7 @@ struct Token { value: Option<(String, Instant)>, } -const TOKEN_MAX_AGE: Duration = Duration::from_secs(50 * 60); +const TOKEN_MAX_AGE: Duration = Duration::from_mins(50); const METADATA_IDENTITY_URL: &str = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity"; @@ -619,7 +618,7 @@ mod tests { } } - /// Source → buffer → destination over file://: first tick publishes everything, a moved + /// Source → buffer → destination over : first tick publishes everything, a moved /// source is pushed on the next tick, an unchanged source is a no-op (no push), a rewound /// source is rejected without `--force` and followed with it. #[tokio::test] diff --git a/crates/walgit-cli/src/repo.rs b/crates/walgit-cli/src/repo.rs index daeeed7..1d17fb8 100644 --- a/crates/walgit-cli/src/repo.rs +++ b/crates/walgit-cli/src/repo.rs @@ -46,7 +46,7 @@ pub async fn run(action: RepoAction, cfg: &Arc) -> Result<()> { println!("(no repositories)"); } else { for id in repos { - println!("{}", id); + println!("{id}"); } } } @@ -58,8 +58,7 @@ pub async fn run(action: RepoAction, cfg: &Arc) -> Result<()> { let manifest = handle.manifest(); let version = handle .manifest_version() - .map(|v| v.to_string()) - .unwrap_or_else(|| "(none)".into()); + .map_or_else(|| "(none)".into(), |v| v.to_string()); println_kv("repo", &id); println_kv("object_format", &manifest.object_format); @@ -102,19 +101,19 @@ async fn policy(action: PolicyAction, store: &walgit_store::DynStore) -> Result< match action { PolicyAction::Get { repo } => { let id = repo_id(&repo)?; - let policy = policy::load(&store, &id).await?; + let policy = policy::load(store, &id).await?; println!("{}", serde_json::to_string_pretty(&policy)?); } PolicyAction::Set { repo, file } => { let id = repo_id(&repo)?; let bytes = std::fs::read(&file)?; let doc: RepoPolicy = serde_json::from_slice(&bytes)?; - policy::save(&store, &id, &doc).await?; + policy::save(store, &id, &doc).await?; info!(repo = %id, "policy saved"); } PolicyAction::Clear { repo } => { let id = repo_id(&repo)?; - policy::clear(&store, &id).await?; + policy::clear(store, &id).await?; info!(repo = %id, "policy cleared"); } } diff --git a/crates/walgit-cli/src/serve.rs b/crates/walgit-cli/src/serve.rs index 777d289..65ecca1 100644 --- a/crates/walgit-cli/src/serve.rs +++ b/crates/walgit-cli/src/serve.rs @@ -106,7 +106,7 @@ async fn compact_loop(registry: Arc, cfg: Arc) { info!("compaction disabled by config, loop exiting"); return; } - let interval = std::time::Duration::from_secs(60); + let interval = std::time::Duration::from_mins(1); loop { tokio::time::sleep(interval).await; if let Err(e) = run_compaction_pass(®istry, &cfg).await { @@ -152,7 +152,7 @@ async fn bundle_loop(bundler: Arc, cfg: Arc) { info!("bundles disabled by config, loop exiting"); return; } - let interval = std::time::Duration::from_secs(60); + let interval = std::time::Duration::from_mins(1); loop { tokio::time::sleep(interval).await; let now = std::time::SystemTime::now(); diff --git a/crates/walgit-cli/src/synth.rs b/crates/walgit-cli/src/synth.rs index d581fb8..6e9945a 100644 --- a/crates/walgit-cli/src/synth.rs +++ b/crates/walgit-cli/src/synth.rs @@ -18,7 +18,7 @@ use anyhow::{Context, Result, bail}; use crate::SynthSize; -/// (commits, files, branches, tags, binary_blobs) +/// (commits, files, branches, tags, `binary_blobs`) fn size_params( size: SynthSize, commits: Option, @@ -105,7 +105,7 @@ pub async fn run( let status = child.wait().context("waiting for git fast-import")?; if !status.success() { - bail!("git fast-import failed (exit {})", status); + bail!("git fast-import failed (exit {status})"); } // Checkout the main branch so it's a working tree. @@ -135,10 +135,7 @@ pub async fn run( .current_dir(&out) .output()?; let head = String::from_utf8_lossy(&head.stdout).trim().to_string(); - println!( - "synth OK: {} commits, {} files, HEAD={}", - n_commits, n_files, head - ); + println!("synth OK: {n_commits} commits, {n_files} files, HEAD={head}"); Ok(()) } @@ -330,7 +327,7 @@ fn message_for(n: u64) -> String { fn generate_file(rng: &mut Rng, file_idx: u64, binary: bool, commit_num: u64) -> (String, Vec) { // Distribute files across directories: dir_0/, dir_1/, ... let dir = file_idx / 100; - let is_binary = binary && (file_idx % 13 == 0); + let is_binary = binary && file_idx.is_multiple_of(13); let ext = if is_binary { "bin" } else { "txt" }; let path = format!("src/dir_{dir}/file_{file_idx:05}.{ext}"); @@ -345,10 +342,15 @@ fn generate_file(rng: &mut Rng, file_idx: u64, binary: bool, commit_num: u64) -> let lines = 3 + (rng.next_u64() % 20) as usize; let mut s = String::with_capacity(lines * 40); for i in 0..lines { - s.push_str(&format!( - "line {i} of file {file_idx} at commit {commit_num}: {:016x}\n", - rng.next_u64() - )); + { + let _ = std::fmt::Write::write_fmt( + &mut s, + format_args!( + "line {i} of file {file_idx} at commit {commit_num}: {:016x}\n", + rng.next_u64() + ), + ); + }; } s.into_bytes() }; diff --git a/crates/walgit-cli/src/wal_cmd.rs b/crates/walgit-cli/src/wal_cmd.rs index cb1810c..beb473d 100644 --- a/crates/walgit-cli/src/wal_cmd.rs +++ b/crates/walgit-cli/src/wal_cmd.rs @@ -31,8 +31,8 @@ pub async fn run(action: WalAction, cfg: &Arc) -> Result<()> { } println!( - "{:<6} {:<10} {:<12} {:<10} {}", - "seq", "kind", "pack", "supersedes", "refs" + "{:<6} {:<10} {:<12} {:<10} refs", + "seq", "kind", "pack", "supersedes" ); for e in &entries { let kind = format!("{:?}", e.kind); @@ -42,7 +42,7 @@ pub async fn run(action: WalAction, cfg: &Arc) -> Result<()> { .map(|p| p.checksum[..12].to_string()) .unwrap_or_default(); let supersedes = e.supersedes.len(); - let ref_count = e.txn.as_ref().map(|t| t.updates.len()).unwrap_or(0); + let ref_count = e.txn.as_ref().map_or(0, |t| t.updates.len()); println!( "{:<6} {:<10} {:<12} {:<10} {}", e.seq, kind, pack, supersedes, ref_count @@ -95,7 +95,7 @@ pub async fn run(action: WalAction, cfg: &Arc) -> Result<()> { println!( "{} ({} bytes) in {:.1}s", out.display(), - std::fs::metadata(&out).map(|m| m.len()).unwrap_or(0), + std::fs::metadata(&out).map_or(0, |m| m.len()), t0.elapsed().as_secs_f64() ); } @@ -148,14 +148,13 @@ pub async fn run(action: WalAction, cfg: &Arc) -> Result<()> { println_kv("writer", &entry.writer); println_kv( "created_at", - &entry - .created_at - .as_ref() - .map(|t| { + entry.created_at.as_ref().map_or_else( + || "(none — predates the field)".into(), + |t| { humantime::format_rfc3339_seconds(walgit_proto::time::to_system(t)) .to_string() - }) - .unwrap_or_else(|| "(none — predates the field)".into()), + }, + ), ); if let Some(pack) = &entry.pack { @@ -214,7 +213,11 @@ pub async fn materialize_at( at_seq: u64, out: &std::path::Path, ) -> Result<()> { - let handle = registry.open(&id).await?; + use walgit_store::ObjectStoreExt; + + use walgit_proto::prost::Message; + + let handle = registry.open(id).await?; // Read log entries up to at_seq and replay into a fresh LocalRepo. if out.exists() { @@ -229,9 +232,7 @@ pub async fn materialize_at( other => bail!("unknown object format in manifest: {other}"), }; - let local = walgit_git::LocalRepo::init(out, &id, format)?; - use walgit_proto::prost::Message; - use walgit_store::ObjectStoreExt; + let local = walgit_git::LocalRepo::init(out, id, format)?; // Start from the newest checkpoint at or before `at_seq` when the // log before it has been folded (min_seq), else from seq 0. @@ -273,7 +274,7 @@ pub async fn materialize_at( match handle.store().get_bytes(&key).await? { Some((_, bytes)) => { let (es, _) = walgit_proto::frame::decode_entries(&bytes)?; - let last = es.last().map(|e| e.seq).unwrap_or(seq); + let last = es.last().map_or(seq, |e| e.seq); found.extend(es); seq = last + 1; } @@ -283,8 +284,7 @@ pub async fn materialize_at( manifest .checkpoint .as_ref() - .map(|c| c.seq) - .unwrap_or(manifest.min_seq) + .map_or(manifest.min_seq, |c| c.seq) ), } } @@ -532,6 +532,6 @@ mod tests { .success() ); // The writer's live copy kept its packs. - assert!(handle.local().packs().unwrap().len() >= 1); + assert!(!handle.local().packs().unwrap().is_empty()); } } diff --git a/crates/walgit-config/src/lib.rs b/crates/walgit-config/src/lib.rs index 9132dc0..6b95f55 100644 --- a/crates/walgit-config/src/lib.rs +++ b/crates/walgit-config/src/lib.rs @@ -11,6 +11,7 @@ pub use std::str::FromStr; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] +#[derive(Default)] pub struct Config { pub server: ServerConfig, pub store: StoreConfig, @@ -48,8 +49,8 @@ pub struct ServerConfig { pub drain_timeout: Duration, /// Max size of a single pushed pack accepted over HTTP. pub max_push_bytes: ByteSize, - /// Roles this instance performs. a serverless host: fronts get ["serve"], the - /// single maintenance instance ["maintain"] (checkpoint / bundle / compact + /// Roles this instance performs. a serverless host: fronts get `["serve"]`, the + /// single maintenance instance `["maintain"]` (checkpoint / bundle / compact /// loops over every repo; `compact` and `bundle` are its sub-roles). Empty = all. pub roles: Vec, pub auth: AuthConfig, @@ -199,7 +200,7 @@ pub enum AuthMode { None, /// Static tokens from the config (`tokens`), bearer or basic. Token, - /// OpenID Connect: browser sign-in through the issuer, ID tokens as bearers, plus + /// `OpenID` Connect: browser sign-in through the issuer, ID tokens as bearers, plus /// walgit-issued access tokens for git — and `tokens` for robots. Oidc, } @@ -338,6 +339,10 @@ pub struct CacheConfig { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] +#[expect( + clippy::struct_excessive_bools, + reason = "Independent configuration switches, not mutually exclusive states" +)] pub struct WalConfig { /// Coalesce concurrent publishes to one repo within this window into one index CAS. #[serde(with = "humantime_serde")] @@ -449,12 +454,12 @@ fn default_all_repos() -> Vec { impl Default for MaintenanceConfig { fn default() -> Self { MaintenanceConfig { - interval: Duration::from_secs(60), + interval: Duration::from_mins(1), checkpoints: true, max_pack_bytes: ByteSize::b(0), disk: MaintainerDisk::Tmpfs, host: None, - fsck_interval: Duration::from_secs(7 * 24 * 3600), + fsck_interval: Duration::from_hours(168), follow_interval: Duration::from_secs(30), } } @@ -469,6 +474,7 @@ impl Default for MaintenanceConfig { /// everywhere, so the edge's read-only fallback (D29) works. /// * **maintain**: the maintainer loop's units (checkpoints, bundles, compaction, /// fsck/repair) — only on hosts with the `maintain` role. +/// /// Placement is by rule, not by capacity: a repo is either this host's or not. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] @@ -530,6 +536,10 @@ pub enum RepackEngine { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] +#[expect( + clippy::struct_excessive_bools, + reason = "Independent configuration switches, not mutually exclusive states" +)] pub struct BundlesConfig { pub enabled: bool, pub strategy: Vec, @@ -683,6 +693,10 @@ pub struct UpstreamConfig { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] +#[expect( + clippy::struct_excessive_bools, + reason = "Independent configuration switches, not mutually exclusive states" +)] pub struct GitConfig { /// Path to the upstream git binary (repack, bundle, optional upload-pack engine). pub binary: PathBuf, @@ -763,7 +777,7 @@ impl Default for EventsConfig { EventsConfig { webhook_url: None, webhook_secret: None, - sweep_interval: Duration::from_secs(300), + sweep_interval: Duration::from_mins(5), } } } @@ -814,6 +828,16 @@ impl Config { /// Only [`SETTINGS_SECTIONS`] may appear; the result is validated like a /// config file. Empty settings = `self` unchanged. pub fn with_settings(&self, settings_toml: &str) -> Result { + fn merge(into: &mut toml::Table, from: &toml::Table) { + for (k, v) in from { + match (into.get_mut(k), v) { + (Some(toml::Value::Table(a)), toml::Value::Table(b)) => merge(a, b), + _ => { + into.insert(k.clone(), v.clone()); + } + } + } + } if settings_toml.trim().is_empty() { return Ok(self.clone()); } @@ -836,16 +860,6 @@ impl Config { ); } let mut doc: toml::Table = toml::Table::try_from(self).context("serializing config")?; - fn merge(into: &mut toml::Table, from: &toml::Table) { - for (k, v) in from { - match (into.get_mut(k), v) { - (Some(toml::Value::Table(a)), toml::Value::Table(b)) => merge(a, b), - _ => { - into.insert(k.clone(), v.clone()); - } - } - } - } merge(&mut doc, &overrides); let cfg: Config = doc.try_into().context("settings: applying")?; cfg.validate() @@ -857,7 +871,7 @@ impl Config { /// never `upstream.token_env` (that name is host-only). pub fn public_settings_toml(&self) -> Result { let mut doc: toml::Table = toml::Table::try_from(self).context("serializing config")?; - doc.retain(|k, _| SETTINGS_SECTIONS.iter().any(|s| *s == k)); + doc.retain(|k, _| SETTINGS_SECTIONS.contains(&k)); if let Some(toml::Value::Table(u)) = doc.get_mut("upstream") { u.remove("token_env"); } @@ -882,7 +896,7 @@ impl Config { CacheMode::Auto => self.maintenance.disk == MaintainerDisk::Ssd, } } - /// Bundle strategies form chains of calendar slots (docs/BUNDLE_URI_DESIGN.md §4): + /// Bundle strategies form chains of calendar slots (`docs/BUNDLE_URI_DESIGN.md` §4): /// every `schedule` is a 6-field UTC cron (or an `@alias`) that parses; an /// incremental names a `base` that exists and whose chain ends in a full /// strategy; each chain has exactly one full root; `keep >= 1` on fulls. @@ -987,7 +1001,7 @@ fn env_placement_overrides(doc: &toml::Table, vars_seen: &[String]) -> Option = vars_seen .iter() .filter_map(|k| k.strip_prefix("WALGIT__PLACEMENT__")) - .map(|k| k.to_ascii_lowercase()) + .map(str::to_ascii_lowercase) .collect(); if keys.is_empty() { return None; @@ -1009,33 +1023,14 @@ pub fn repo_listed(list: &[String], owner: &str, name: &str) -> bool { }) } -impl Default for Config { - fn default() -> Self { - Config { - server: ServerConfig::default(), - store: StoreConfig::default(), - cache: CacheConfig::default(), - wal: WalConfig::default(), - compaction: CompactionConfig::default(), - maintenance: MaintenanceConfig::default(), - bundles: BundlesConfig::default(), - placement: PlacementConfig::default(), - lfs: LfsConfig::default(), - upstream: UpstreamConfig::default(), - git: GitConfig::default(), - telemetry: TelemetryConfig::default(), - events: EventsConfig::default(), - } - } -} impl Default for ServerConfig { fn default() -> Self { ServerConfig { - listen: "127.0.0.1:8080".parse().unwrap(), + listen: std::net::SocketAddr::from(([127, 0, 0, 1], 8080)), http2: true, max_concurrent_requests: 512, max_concurrent_per_repo: 64, - request_timeout: Duration::from_secs(3600), + request_timeout: Duration::from_hours(1), drain_timeout: Duration::from_secs(20), max_push_bytes: ByteSize::gib(64), roles: vec![], @@ -1063,8 +1058,8 @@ impl Default for AuthConfig { admin_emails: vec![], admin_domains: vec![], session_secret: None, - session_ttl: Duration::from_secs(30 * 24 * 3600), - access_token_ttl: Duration::from_secs(90 * 24 * 3600), + session_ttl: Duration::from_hours(720), + access_token_ttl: Duration::from_hours(2160), oauth_client_id: None, oauth_client_secret: None, } @@ -1113,7 +1108,7 @@ impl Default for CacheConfig { mode: CacheMode::Auto, max_bytes: ByteSize::gib(20), disk_high_watermark: 0.9, - evict_idle_after: Duration::from_secs(6 * 3600), + evict_idle_after: Duration::from_hours(6), prewarm: vec![], prewarm_parallelism: 2, prewarm_ready_timeout: Duration::ZERO, @@ -1136,7 +1131,7 @@ impl Default for WalConfig { push_broker_token: None, push_broker_buffer_bytes: ByteSize::mib(64), snapshot_every_entries: 256, - checkpoint_interval: Duration::from_secs(3600), + checkpoint_interval: Duration::from_hours(1), checkpoint_tail_bytes: ByteSize::mib(8), cas_max_retries: 16, fsck_objects: true, @@ -1155,8 +1150,8 @@ impl Default for CompactionConfig { factor: 2, trigger_packs: 16, trigger_bytes: ByteSize::gib(1), - lease_ttl: Duration::from_secs(600), - retention_superseded: Duration::from_secs(7 * 24 * 3600), + lease_ttl: Duration::from_mins(10), + retention_superseded: Duration::from_hours(168), engine: RepackEngine::Git, } } @@ -1205,7 +1200,7 @@ impl Default for BundlesConfig { }, ], serve_via: BundleServe::Proxy, - signed_url_ttl: Duration::from_secs(3600), + signed_url_ttl: Duration::from_hours(1), advertise: true, advertise_filtered: false, require: Vec::new(), @@ -1222,7 +1217,7 @@ impl Default for LfsConfig { LfsConfig { enabled: true, serve_via: BundleServe::Proxy, - signed_url_ttl: Duration::from_secs(3600), + signed_url_ttl: Duration::from_hours(1), max_object_bytes: ByteSize::gib(16), } } @@ -1304,8 +1299,8 @@ impl Config { continue; }; vars_seen.push(k.clone()); - let path: Vec = rest.split("__").map(|s| s.to_ascii_lowercase()).collect(); - if path.is_empty() || path.iter().any(|p| p.is_empty()) { + let path: Vec = rest.split("__").map(str::to_ascii_lowercase).collect(); + if path.is_empty() || path.iter().any(std::string::String::is_empty) { continue; } let value: toml::Value = v @@ -1320,16 +1315,19 @@ impl Config { path: &[String], value: toml::Value, ) -> std::result::Result<(), String> { - if path.len() == 1 { - cur.insert(path[0].clone(), value); + let Some((key, rest)) = path.split_first() else { + return Err("empty configuration path".into()); + }; + if rest.is_empty() { + cur.insert(key.clone(), value); return Ok(()); } let next = cur - .entry(path[0].clone()) - .or_insert_with(|| toml::Value::Table(Default::default())) + .entry(key.clone()) + .or_insert_with(|| toml::Value::Table(toml::Table::new())) .as_table_mut() - .ok_or_else(|| format!("{} is not a table", path[0]))?; - set(next, &path[1..], value) + .ok_or_else(|| format!("{key} is not a table"))?; + set(next, rest, value) } match set(&mut trial, &path, value) { Err(why) => Some(why), @@ -1342,12 +1340,11 @@ impl Config { }), } }; - match bad { - Some(why) => ignored.push((k, why)), - None => { - doc = trial; - touched = true; - } + if let Some(why) = bad { + ignored.push((k, why)); + } else { + doc = trial; + touched = true; } } // `[placement]` is a host fact set as a GROUP: any WALGIT__PLACEMENT__* override @@ -1371,10 +1368,10 @@ impl Config { self.server.listen.set_port(port); // Standalone / `dev server`: public_url is the origin the browser hits. Keep its // port in lockstep with PORT. A real public_url is left alone. - if let Some(u) = self.server.public_url.as_mut() { - if origin_is_loopback(u) { - *u = rewrite_origin_port(u, port); - } + if let Some(u) = self.server.public_url.as_mut() + && origin_is_loopback(u) + { + *u = rewrite_origin_port(u, port); } } Ok(ignored) @@ -1555,7 +1552,7 @@ impl Config { names.contains(b.as_str()), "bundle strategy {} base {b} does not exist", s.name - ) + ); } (BundleKind::Full, Some(_)) => { anyhow::bail!("bundle strategy {} is full but has a base", s.name) @@ -1622,7 +1619,7 @@ impl Config { } let mut v: Vec = ["localhost", "*.localhost", "127.0.0.1", "::1"] .iter() - .map(|s| s.to_string()) + .map(std::string::ToString::to_string) .collect(); if let Some(u) = &self.server.public_url { let host = u @@ -1634,8 +1631,7 @@ impl Config { .trim_start_matches('['); let host = host .rsplit_once(']') - .map(|(h, _)| h) - .unwrap_or_else(|| host.split(':').next().unwrap_or(host)); + .map_or_else(|| host.split(':').next().unwrap_or(host), |(h, _)| h); if !host.is_empty() && !v.iter().any(|h| h == host) { v.push(host.to_string()); } @@ -1655,10 +1651,9 @@ fn origin_host(origin: &str) -> &str { let rest = origin .trim_end_matches('/') .split_once("://") - .map(|(_, r)| r) - .unwrap_or(origin); + .map_or(origin, |(_, r)| r); if let Some(inside) = rest.strip_prefix('[') { - return inside.split_once(']').map(|(h, _)| h).unwrap_or(inside); + return inside.split_once(']').map_or(inside, |(h, _)| h); } rest.split([':', '/']).next().unwrap_or(rest) } @@ -1675,8 +1670,7 @@ fn rewrite_origin_port(origin: &str, port: u16) -> String { }; let host = if rest.starts_with('[') { rest.split_once(']') - .map(|(h, _)| format!("{h}]")) - .unwrap_or_else(|| rest.to_string()) + .map_or_else(|| rest.to_string(), |(h, _)| format!("{h}]")) } else { rest.split([':', '/']).next().unwrap_or(rest).to_string() }; @@ -1754,7 +1748,7 @@ mod tests { /// `[placement]` is set as a group: one PLACEMENT env key replaces the whole /// section (unset keys = defaults), never merges with the file's values. - /// The SSD host 2026-08-21 07:00Z: the baked toml's serve_exclude = ["acme/monorepo"] + /// The SSD host 2026-08-21 07:00Z: the baked toml's `serve_exclude` = `["acme/monorepo"]` /// leaked under an env that set only MAINTAIN* → the host refused its own repo. #[test] fn env_placement_override_replaces_the_whole_section() { @@ -1868,13 +1862,13 @@ mod tests { base.store.bucket = "b".into(); let eff = base .with_settings( - r#" + r" [bundles] min_commits = 3 main_only = false [maintenance] checkpoints = false -"#, +", ) .unwrap(); assert_eq!(eff.bundles.min_commits, 3); @@ -1945,10 +1939,7 @@ audiences = ["walgit-cli", "https://git.example.com"] assert!(err.to_string().contains("session_secret"), "{err}"); let ok = Config::parse("[store]\nbucket = \"b\"\n[server.auth]\nmode = \"oidc\"\nissuer = \"https://login.example.com\"\nanonymous_read = false\nallowed_domains = [\"example.com\"]\noauth_client_id = \"x\"\noauth_client_secret = \"y\"\nsession_secret = \"0123456789abcdef0123456789abcdef\"\n").unwrap(); assert_eq!(ok.server.auth.issuer, "https://login.example.com"); - assert_eq!( - ok.server.auth.access_token_ttl, - Duration::from_secs(90 * 86400) - ); + assert_eq!(ok.server.auth.access_token_ttl, Duration::from_hours(2160)); let err = Config::parse( "[store]\nbucket = \"b\"\n[server]\nlisten = \"0.0.0.0:8080\"\n[server.auth]\nmode = \"none\"\n", ) @@ -1967,7 +1958,7 @@ webhook_secret = "s" "#, ) .unwrap(); - assert_eq!(c.events.sweep_interval, Duration::from_secs(60)); + assert_eq!(c.events.sweep_interval, Duration::from_mins(1)); assert_eq!(c.events.webhook_secret.as_deref(), Some("s")); let err = Config::parse("[events]\nwebhook_url = \"ftp://x\"\n").unwrap_err(); assert!(err.to_string().contains("webhook_url"), "{err}"); diff --git a/crates/walgit-git/src/follow.rs b/crates/walgit-git/src/follow.rs index 2714e25..35f76f7 100644 --- a/crates/walgit-git/src/follow.rs +++ b/crates/walgit-git/src/follow.rs @@ -42,11 +42,11 @@ impl FetchedDelta { /// Fetch `refs` from `upstream` into the scratch for `(owner, name)` under `dir`, /// negotiating from `have` (`ref → oid` we hold; missing = fetch its history). -pub async fn fetch_refs( +pub async fn fetch_refs( upstream: &str, token: Option<&str>, serving_objects: &Path, - have: &HashMap, + have: &HashMap, refs: &[String], scratch: &Path, ) -> Result { @@ -121,8 +121,18 @@ pub async fn fetch_refs( let mut input = String::new(); for r in refs { match have.get(r) { - Some(oid) => input.push_str(&format!("update {} {oid}\n", follow_ref(r))), - None => input.push_str(&format!("delete {}\n", follow_ref(r))), + Some(oid) => { + let _ = std::fmt::Write::write_fmt( + &mut input, + format_args!("update {} {oid}\n", follow_ref(r)), + ); + } + None => { + let _ = std::fmt::Write::write_fmt( + &mut input, + format_args!("delete {}\n", follow_ref(r)), + ); + } } } let mut child = git(&["update-ref", "--stdin"]) @@ -131,7 +141,10 @@ pub async fn fetch_refs( .map_err(GitError::Io)?; { use tokio::io::AsyncWriteExt; - let mut stdin = child.stdin.take().expect("stdin"); + let mut stdin = child + .stdin + .take() + .ok_or_else(|| std::io::Error::other("git stdin unavailable"))?; stdin .write_all(input.as_bytes()) .await diff --git a/crates/walgit-git/src/lib.rs b/crates/walgit-git/src/lib.rs index d48fa36..1d277ef 100644 --- a/crates/walgit-git/src/lib.rs +++ b/crates/walgit-git/src/lib.rs @@ -1,6 +1,6 @@ //! Local git repository engine: gix in-process for odb/refs/revwalk/pack //! generation; upstream git subprocess for ingest (`index-pack`), repack, -//! bundle, and the selectable Engine::Git upload-pack fallback. See AGENTS.md +//! bundle, and the selectable `Engine::Git` upload-pack fallback. See AGENTS.md //! D2 and docs/CONTRACT.md walgit-git. pub mod follow; @@ -64,6 +64,10 @@ fn ge(e: E) -> GitError { /// Reject ref names that would inject `git update-ref --stdin` commands or /// poison packed-refs (newlines, NULs, git-illegal bytes). +#[expect( + clippy::case_sensitive_file_extension_comparisons, + reason = "Git forbids exactly the case-sensitive .lock suffix" +)] pub fn validate_ref_name(name: &str) -> Result<(), GitError> { if name == "HEAD" { return Ok(()); @@ -163,7 +167,7 @@ impl RepoId { &self.name } - /// `repos///` (walgit_proto::keys::repo_prefix). + /// `repos///` (`walgit_proto::keys::repo_prefix`). pub fn store_prefix(&self) -> String { walgit_proto::keys::repo_prefix(&self.owner, &self.name) } @@ -259,7 +263,6 @@ impl From for ObjectFormat { impl From for ObjectFormat { fn from(k: gix_hash::Kind) -> Self { match k { - gix_hash::Kind::Sha1 => ObjectFormat::Sha1, gix_hash::Kind::Sha256 => ObjectFormat::Sha256, _ => ObjectFormat::Sha1, } @@ -378,13 +381,17 @@ impl LsRefsLine { /// ` peeled:`, then a trailing newline. pub fn render(&self, args: &LsRefsArgs) -> String { let mut s = format!("{} {}", self.oid, self.name); - if args.symrefs || self.oid == "unborn" { - if let Some(t) = &self.symref_target { - s.push_str(&format!(" symref-target:{t}")); - } + if (args.symrefs || self.oid == "unborn") + && let Some(t) = &self.symref_target + { + { + let _ = std::fmt::Write::write_fmt(&mut s, format_args!(" symref-target:{t}")); + }; } if args.peel && !self.peeled.is_empty() { - s.push_str(&format!(" peeled:{}", self.peeled)); + { + let _ = std::fmt::Write::write_fmt(&mut s, format_args!(" peeled:{}", self.peeled)); + }; } s.push('\n'); s @@ -418,6 +425,10 @@ impl Service { } #[derive(Debug, Clone)] +#[expect( + clippy::struct_excessive_bools, + reason = "Independent Git protocol capabilities and request flags" +)] pub struct UploadPackRequest { pub wants: Vec, pub haves: Vec, @@ -511,7 +522,8 @@ impl RefView { .refs .binary_search_by(|r| r.name.as_str().cmp(name)) .ok() - .map(|i| self.base.refs[i].oid.clone()) + .and_then(|i| self.base.refs.get(i)) + .map(|r| r.oid.clone()) } pub fn head_target(&self) -> &str { self.head_target @@ -578,7 +590,7 @@ fn refs_key(path: &Path, generation: u64) -> RefsKey { let head = std::fs::metadata(path.join("HEAD")).ok(); RefsKey { generation, - packed_len: packed.as_ref().map(|m| m.len()).unwrap_or(0), + packed_len: packed.as_ref().map_or(0, std::fs::Metadata::len), packed_mtime: packed.and_then(|m| m.modified().ok()), head_mtime: head.and_then(|m| m.modified().ok()), } @@ -593,7 +605,7 @@ impl LocalRepo { /// Create a bare repo at `//.git`. pub fn init(root: &Path, id: &RepoId, format: ObjectFormat) -> Result { let path = id.local_dir(root); - std::fs::create_dir_all(path.parent().unwrap_or(root)).map_err(|e| GitError::Io(e))?; + std::fs::create_dir_all(path.parent().unwrap_or(root)).map_err(GitError::Io)?; // `git init --bare [--object-format=...] `. let mut cmd = std::process::Command::new("git"); cmd.arg("init").arg("--bare"); @@ -710,7 +722,7 @@ impl LocalRepo { let t = std::time::Instant::now(); // `iter()` snapshots the store with all indices loaded. let _ = repo.objects.iter(); - let ms = t.elapsed().as_millis() as u64; + let ms = u64::try_from(t.elapsed().as_millis()).unwrap_or(u64::MAX); if ms > 200 { tracing::info!(repo = %self.inner.path.display(), ms, "odb indices loaded"); } @@ -778,7 +790,7 @@ impl LocalRepo { .open(&candidate) { Ok(file) => break (candidate, file), - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} Err(e) => return Err(GitError::Io(e)), } }; @@ -797,22 +809,25 @@ impl LocalRepo { } empty_check = false; total += n as u64; - if let Some(max) = opts.max_bytes { - if total > max { - drop( - tokio::fs::remove_file(&tmp_path) - .instrument(span.clone()) - .await, - ); - return Err(GitError::InvalidInput(format!( - "pack exceeds max_bytes {max}" - ))); - } + if let Some(max) = opts.max_bytes + && total > max + { + drop( + tokio::fs::remove_file(&tmp_path) + .instrument(span.clone()) + .await, + ); + return Err(GitError::InvalidInput(format!( + "pack exceeds max_bytes {max}" + ))); } - tmp.write_all(&buf[..n]) - .instrument(span.clone()) - .await - .map_err(GitError::Io)?; + tmp.write_all( + buf.get(..n) + .ok_or_else(|| std::io::Error::other("read exceeded buffer"))?, + ) + .instrument(span.clone()) + .await + .map_err(GitError::Io)?; } // tokio's File buffers writes in a background blocking task and does // NOT flush on drop: without this the tail of the pack may be missing @@ -883,8 +898,8 @@ impl LocalRepo { let _ = std::fs::remove_file(pack_path.with_extension("rev")); return Ok(None); } - let pack_size = std::fs::metadata(&pack_path).map(|m| m.len()).unwrap_or(0); - let idx_size = std::fs::metadata(&idx_path).map(|m| m.len()).unwrap_or(0); + let pack_size = std::fs::metadata(&pack_path).map_or(0, |m| m.len()); + let idx_size = std::fs::metadata(&idx_path).map_or(0, |m| m.len()); self.refresh_async() .instrument(tracing::info_span!(parent: &span, "git.ingest_pack.refresh")) .await?; @@ -907,12 +922,18 @@ impl LocalRepo { ) -> Result<(), GitError> { let pack_dir = self.objects_pack_dir(); std::fs::create_dir_all(&pack_dir).map_err(GitError::Io)?; - let dst_pack = pack_dir.join(pack.file_name().unwrap()); - let dst_idx = pack_dir.join(idx.file_name().unwrap()); + let dst_pack = pack_dir.join(pack.file_name().ok_or_else(|| { + GitError::InvalidInput(format!("missing filename: {}", pack.display())) + })?); + let dst_idx = pack_dir.join(idx.file_name().ok_or_else(|| { + GitError::InvalidInput(format!("missing filename: {}", idx.display())) + })?); rename_atomic(pack, &dst_pack)?; rename_atomic(idx, &dst_idx)?; for e in extra { - let dst = pack_dir.join(e.file_name().unwrap()); + let dst = pack_dir.join(e.file_name().ok_or_else(|| { + GitError::InvalidInput(format!("missing filename: {}", e.display())) + })?); rename_atomic(e, &dst)?; } self.refresh_async().await?; @@ -978,15 +999,19 @@ impl LocalRepo { if !name.starts_with("pack-") || !name.ends_with(".pack") { continue; } - let hex = &name["pack-".len()..name.len() - ".pack".len()]; - let checksum = match gix_hash::ObjectId::from_hex(hex.as_bytes()) { - Ok(o) => o, - Err(_) => continue, + let Some(hex) = name + .strip_prefix("pack-") + .and_then(|n| n.strip_suffix(".pack")) + else { + continue; + }; + let Ok(checksum) = gix_hash::ObjectId::from_hex(hex.as_bytes()) else { + continue; }; let pack_path = ent.path(); let idx_path = pack_path.with_extension("idx"); - let pack_size = std::fs::metadata(&pack_path).map(|m| m.len()).unwrap_or(0); - let idx_size = std::fs::metadata(&idx_path).map(|m| m.len()).unwrap_or(0); + let pack_size = std::fs::metadata(&pack_path).map_or(0, |m| m.len()); + let idx_size = std::fs::metadata(&idx_path).map_or(0, |m| m.len()); let object_count = idx_object_count(&idx_path).unwrap_or(0); let has_rev = pack_path.with_extension("rev").exists(); let has_bitmap = pack_path.with_extension("bitmap").exists(); @@ -1022,7 +1047,7 @@ impl LocalRepo { } /// The refs, parsed once and shared: `packed-refs` of a 500 k-ref repo is - /// 34 MB and read_refs also peels every tag — 1–2 s per call, which every + /// 34 MB and `read_refs` also peels every tag — 1–2 s per call, which every /// `ls-refs` (prefix or not) paid (2026-08-21, test/refs500k on a serverless host). /// Valid until a ref writer in this process bumps the generation or /// `packed-refs`/`HEAD` change on disk (two stats per call). Sorted by name. @@ -1042,11 +1067,11 @@ impl LocalRepo { // Fold the pushes applied since the last materialization: one copy of the // vector for all of them, no parsing, no object reads. let t = std::time::Instant::now(); - let patched = self.patch_snapshot(&c.data, &c.pending); + let patched = self.patch_snapshot(&c.data, &c.pending)?; c.data = Arc::new(patched); c.pending.clear(); if c.data.refs.len() >= 10_000 { - tracing::debug!(repo = %self.inner.id, refs = c.data.refs.len(), ms = t.elapsed().as_millis() as u64, "refs cache materialized from pending txns"); + tracing::debug!(repo = %self.inner.id, refs = c.data.refs.len(), ms = u64::try_from(t.elapsed().as_millis()).unwrap_or(u64::MAX), "refs cache materialized from pending txns"); } } return Ok(c.data.clone()); @@ -1058,7 +1083,7 @@ impl LocalRepo { .refs_parses .fetch_add(1, std::sync::atomic::Ordering::Relaxed); if data.refs.len() >= 10_000 { - tracing::debug!(repo = %self.inner.id, refs = data.refs.len(), ms = t.elapsed().as_millis() as u64, "refs parsed into the cache"); + tracing::debug!(repo = %self.inner.id, refs = data.refs.len(), ms = u64::try_from(t.elapsed().as_millis()).unwrap_or(u64::MAX), "refs parsed into the cache"); } *self.inner.refs_cache.lock() = Some(RefsCached { key, @@ -1173,14 +1198,14 @@ impl LocalRepo { return Err(GitError::RefConflict { name: u.name.clone(), expected: u.old_oid.clone(), - actual: current.to_string(), + actual: current.clone(), }); } } else if old != cur { return Err(GitError::RefConflict { name: u.name.clone(), expected: u.old_oid.clone(), - actual: current.to_string(), + actual: current.clone(), }); } } @@ -1205,16 +1230,41 @@ impl LocalRepo { if new_zero { // delete if check_old && !old_zero { - input.push_str(&format!("delete {} {}\n", u.name, u.old_oid)); + { + let _ = std::fmt::Write::write_fmt( + &mut input, + format_args!("delete {} {}\n", u.name, u.old_oid), + ); + }; } else { - input.push_str(&format!("delete {}\n", u.name)); + { + let _ = std::fmt::Write::write_fmt( + &mut input, + format_args!("delete {}\n", u.name), + ); + }; } } else if check_old && old_zero { - input.push_str(&format!("create {} {}\n", u.name, u.new_oid)); + { + let _ = std::fmt::Write::write_fmt( + &mut input, + format_args!("create {} {}\n", u.name, u.new_oid), + ); + }; } else if check_old && !old_zero { - input.push_str(&format!("update {} {} {}\n", u.name, u.new_oid, u.old_oid)); + { + let _ = std::fmt::Write::write_fmt( + &mut input, + format_args!("update {} {} {}\n", u.name, u.new_oid, u.old_oid), + ); + }; } else { - input.push_str(&format!("update {} {}\n", u.name, u.new_oid)); + { + let _ = std::fmt::Write::write_fmt( + &mut input, + format_args!("update {} {}\n", u.name, u.new_oid), + ); + }; } } @@ -1230,7 +1280,10 @@ impl LocalRepo { .spawn() .and_then(|mut c| { { - let stdin = c.stdin.as_mut().unwrap(); + let stdin = c + .stdin + .as_mut() + .ok_or_else(|| std::io::Error::other("git stdin unavailable"))?; stdin.write_all(input.as_bytes())?; } c.wait_with_output() @@ -1295,14 +1348,14 @@ impl LocalRepo { &self, base: &RefSnapshotData, txns: &[walgit_proto::v1::RefTransaction], - ) -> RefSnapshotData { + ) -> Result { let mut refs = base.refs.clone(); let mut head_target = base.head_target.clone(); let mut repo: Option = None; for u in txns.iter().flat_map(|t| t.updates.iter()) { if !u.new_symbolic_target.is_empty() { if u.name == "HEAD" { - head_target = u.new_symbolic_target.clone(); + head_target.clone_from(&u.new_symbolic_target); } continue; } @@ -1316,12 +1369,16 @@ impl LocalRepo { (pos, false) => { let mut peeled = u.new_peeled.clone(); if peeled.is_empty() && u.name.starts_with("refs/tags/") { - let r = repo.get_or_insert_with(|| { - gix::Repository::from( - &gix::ThreadSafeRepository::open(&self.inner.path) - .expect("repo open"), + if repo.is_none() { + repo = Some(gix::Repository::from( + &gix::ThreadSafeRepository::open(&self.inner.path).map_err(ge)?, + )); + } + let r = repo.as_ref().ok_or_else(|| { + GitError::InvalidInput( + "repository unavailable while peeling tag".into(), ) - }); + })?; if let Ok(oid) = gix_hash::ObjectId::from_hex(u.new_oid.as_bytes()) { peeled = peel_tag(r, oid) .map(|p| p.to_hex().to_string()) @@ -1334,13 +1391,17 @@ impl LocalRepo { peeled, }; match pos { - Ok(i) => refs[i] = entry, + Ok(i) => { + *refs.get_mut(i).ok_or_else(|| { + GitError::InvalidInput("ref search index out of bounds".into()) + })? = entry; + } Err(i) => refs.insert(i, entry), } } } } - RefSnapshotData { refs, head_target } + Ok(RefSnapshotData { refs, head_target }) } /// Replace ALL refs + HEAD by writing `packed-refs` directly and removing @@ -1354,9 +1415,17 @@ impl LocalRepo { let mut refs = snap.refs.clone(); refs.sort_by(|a, b| a.name.cmp(&b.name)); for r in &refs { - content.push_str(&format!("{} {}\n", r.oid, r.name)); + { + let _ = std::fmt::Write::write_fmt( + &mut content, + format_args!("{} {}\n", r.oid, r.name), + ); + }; if !r.peeled.is_empty() { - content.push_str(&format!("^{}\n", r.peeled)); + { + let _ = + std::fmt::Write::write_fmt(&mut content, format_args!("^{}\n", r.peeled)); + }; } } // Atomic write. @@ -1412,7 +1481,7 @@ impl LocalRepo { validate_ref_update(u)?; if !u.new_symbolic_target.is_empty() { if u.name == "HEAD" { - head_target = u.new_symbolic_target.clone(); + head_target.clone_from(&u.new_symbolic_target); } continue; } @@ -1478,8 +1547,14 @@ impl LocalRepo { .inner .path .join("objects") - .join(&hex[..2]) - .join(&hex[2..]); + .join( + hex.get(..2) + .ok_or_else(|| GitError::InvalidInput("short object ID".into()))?, + ) + .join( + hex.get(2..) + .ok_or_else(|| GitError::InvalidInput("short object ID".into()))?, + ); if path.exists() { return Ok(()); } @@ -1492,14 +1567,14 @@ impl LocalRepo { ); store .write_buf_with_known_id(kind, data, oid.to_owned()) - .map_err(|e| GitError::Gix(e))?; + .map_err(GitError::Gix)?; Ok(()) } - /// Every object reachable from tips exists. When stop_at_existing_refs, + /// Every object reachable from tips exists. When `stop_at_existing_refs`, /// objects already reachable from current refs are assumed present and - /// only the new set is verified. Uses gix revwalk with .with_hidden( - /// existing ref tips) for commit traversal and gix_traverse::tree + /// only the new set is verified. Uses gix revwalk with .`with_hidden`( + /// existing ref tips) for commit traversal and `gix_traverse::tree` /// breadthfirst for tree traversal with a seen-set. pub fn check_connectivity( &self, @@ -1580,10 +1655,10 @@ impl LocalRepo { let mut out = Vec::with_capacity(snap.refs.len()); for r in &snap.refs { // Prefer the pre-peeled oid for tags; otherwise peel cheaply via the odb. - let candidate = if !r.peeled.is_empty() { - r.peeled.as_str() - } else { + let candidate = if r.peeled.is_empty() { r.oid.as_str() + } else { + r.peeled.as_str() }; let Ok(oid) = gix_hash::ObjectId::from_hex(candidate.as_bytes()) else { continue; @@ -1622,7 +1697,7 @@ impl LocalRepo { if !seen.insert(cid) { continue; } - if !repo.has_object(&cid) { + if !repo.has_object(cid) { return Err(GitError::MissingObject { oid: cid.to_hex().to_string(), }); @@ -1632,9 +1707,9 @@ impl LocalRepo { .objects .find_commit_iter(&cid, &mut buf) .map_err(|e| GitError::Gix(Box::new(e)))?; - let tree_id = commit.tree_id().map_err(|e| ge(e))?; + let tree_id = commit.tree_id().map_err(ge)?; if seen.insert(tree_id) { - if !repo.has_object(&tree_id) { + if !repo.has_object(tree_id) { return Err(GitError::MissingObject { oid: tree_id.to_hex().to_string(), }); @@ -1686,9 +1761,9 @@ impl LocalRepo { /// protocol v2 fetch with in-process gix pack generation. Handles /// negotiation (ACK common haves that exist, NAK, ready), shallow-info /// (deepen by depth), wanted-refs, then generates the packfile via - /// gix_pack::data::output (count + entries with delta reuse from on-disk + /// `gix_pack::data::output` (count + entries with delta reuse from on-disk /// packs), framed in sideband-64k (channel 1; progress on 2 unless - /// no_progress) with a final flush. UploadPackStats populated with object + /// `no_progress`) with a final flush. `UploadPackStats` populated with object /// count and byte count. pub async fn upload_pack( &self, @@ -1756,7 +1831,8 @@ impl LocalRepo { snap.refs .binary_search_by(|r| r.name.as_str().cmp(head_target.as_str())) .ok() - .map(|i| snap.refs[i].oid.clone()) + .and_then(|i| snap.refs.get(i)) + .map(|r| r.oid.clone()) }; // Prefix selection is O(log n + k) over the name-sorted list: each // prefix is one range (binary search for its start, scan while it @@ -1770,7 +1846,11 @@ impl LocalRepo { .map(|p| { let start = snap.refs.partition_point(|r| r.name.as_str() < p.as_str()); let mut end = start; - while end < snap.refs.len() && snap.refs[end].name.starts_with(p.as_str()) { + while snap + .refs + .get(end) + .is_some_and(|r| r.name.starts_with(p.as_str())) + { end += 1; } (start, end) @@ -1782,7 +1862,14 @@ impl LocalRepo { for (a, b) in ranges { let a = a.max(cursor); if a < b { - out.extend(snap.refs[a..b].iter()); + out.extend( + snap.refs + .get(a..b) + .ok_or_else(|| { + GitError::InvalidInput("ref prefix range out of bounds".into()) + })? + .iter(), + ); cursor = b; } } @@ -1832,7 +1919,7 @@ impl LocalRepo { pub fn advertise_refs_v0(&self, service: Service, out: &mut Vec) -> Result<(), GitError> { let snap = self.refs()?; let caps = capabilities_for(service, self.inner.format); - let caps_line = format!("\0{}\n", caps); + let caps_line = format!("\0{caps}\n"); if snap.refs.is_empty() { // No refs: emit the capabilities line with a zero id and @@ -1861,16 +1948,16 @@ impl LocalRepo { } // Include HEAD if it has a resolvable target and isn't already the // first advertised ref (upload-pack advertises HEAD). - if !head_target.is_empty() && service == Service::UploadPack { - if let Some(oid) = snap + if !head_target.is_empty() + && service == Service::UploadPack + && let Some(oid) = snap .refs .iter() .find(|r| r.name == head_target) .map(|r| r.oid.clone()) - { - let head_line = format!("{oid} HEAD\n"); - pkt::encode_data(out, head_line.as_bytes()); - } + { + let head_line = format!("{oid} HEAD\n"); + pkt::encode_data(out, head_line.as_bytes()); } } pkt::encode_flush(out); @@ -1950,7 +2037,7 @@ impl LocalRepo { } } } - let arg_refs: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); + let arg_refs: Vec<&str> = args.iter().map(std::string::String::as_str).collect(); let out = self.git(&arg_refs).await?; if !out.status.success() { return Err(GitError::Subprocess { @@ -2026,7 +2113,7 @@ impl LocalRepo { let po_stdout: Stdio = po .stdout .take() - .expect("stdout") + .ok_or_else(|| std::io::Error::other("git stdout unavailable"))? .try_into() .map_err(GitError::Io)?; let ip = tokio::process::Command::new("git") @@ -2048,7 +2135,10 @@ impl LocalRepo { .map_err(GitError::Io)?; { use tokio::io::AsyncWriteExt; - let mut stdin = po.stdin.take().expect("stdin"); + let mut stdin = po + .stdin + .take() + .ok_or_else(|| std::io::Error::other("git stdin unavailable"))?; stdin .write_all(revs.as_bytes()) .await @@ -2173,17 +2263,20 @@ impl LocalRepo { .map(|p| format!("pack-{}.idx", p.checksum)) .collect(); for h in &history { - if let Some(base) = &h.history_of { - if let Some(b) = packs.iter().find(|p| &p.checksum.to_string() == base) { - let n = format!("pack-{}.idx", b.checksum); - if !names.contains(&n) { - names.push(n); - } + if let Some(base) = &h.history_of + && let Some(b) = packs.iter().find(|p| &p.checksum.to_string() == base) + { + let n = format!("pack-{}.idx", b.checksum); + if !names.contains(&n) { + names.push(n); } } } - let preferred = names[0].clone(); - let input = names.iter().map(|n| format!("{n}\n")).collect::(); + let preferred = names + .first() + .ok_or_else(|| GitError::InvalidInput("no pack names".into()))? + .clone(); + let input = format!("{}\n", names.join("\n")); let out = std::process::Command::new("git") .current_dir(&self.inner.path) .env("GIT_DIR", &self.inner.path) @@ -2198,7 +2291,10 @@ impl LocalRepo { .stderr(Stdio::piped()) .spawn() .and_then(|mut c| { - c.stdin.take().unwrap().write_all(input.as_bytes())?; + c.stdin + .take() + .ok_or_else(|| std::io::Error::other("git stdin unavailable"))? + .write_all(input.as_bytes())?; c.wait_with_output() }) .map_err(GitError::Io)?; @@ -2255,7 +2351,7 @@ impl LocalRepo { let tmp = dst.with_extension("commit-graph.tmp"); std::fs::copy(&src, &tmp).map_err(GitError::Io)?; rename_atomic(&tmp, &dst)?; - Ok(std::fs::metadata(&dst).map(|m| m.len()).unwrap_or(0)) + Ok(std::fs::metadata(&dst).map_or(0, |m| m.len())) } /// Hashes listed in `objects/info/commit-graphs/commit-graph-chain` @@ -2285,7 +2381,7 @@ impl LocalRepo { } let hash = commit_graph_layer_hash(&side)?; let chain = self.commit_graph_chain()?; - if chain.first().map(|h| h == &hash).unwrap_or(false) { + if chain.first().is_some_and(|h| h == &hash) { return Ok(true); } let dir = self.commit_graphs_dir(); @@ -2335,7 +2431,12 @@ impl LocalRepo { } let mut input = String::new(); for p in packs { - input.push_str(&format!("pack-{}.idx\n", p.to_hex())); + { + let _ = std::fmt::Write::write_fmt( + &mut input, + format_args!("pack-{}.idx\n", p.to_hex()), + ); + }; } let mut args = vec!["write", "--split", "--stdin-packs"]; if changed_paths { @@ -2387,7 +2488,7 @@ impl LocalRepo { stderr: String::from_utf8_lossy(&bundle_out.stderr).into_owned(), }); } - let size = std::fs::metadata(out).map(|m| m.len()).unwrap_or(0); + let size = std::fs::metadata(out).map_or(0, |m| m.len()); let pack_offset = locate_pack_offset(out).unwrap_or(size); Ok(BundleInfo { size, pack_offset }) } @@ -2477,7 +2578,7 @@ impl LocalRepo { /// In-process gix upload-pack for protocol v2 fetch. Builds the response /// sections (acknowledgments, shallow-info, wanted-refs, packfile) and - /// generates the pack using gix_pack::data::output. + /// generates the pack using `gix_pack::data::output`. async fn upload_pack_gix( &self, req: UploadPackRequest, @@ -2503,7 +2604,7 @@ impl LocalRepo { stdin_bytes: &[u8], ) -> Result { let path = self.inner.path.clone(); - let args: Vec = args.iter().map(|s| s.to_string()).collect(); + let args: Vec = args.iter().map(std::string::ToString::to_string).collect(); let stdin_bytes: Vec = stdin_bytes.to_vec(); let cmd_name = cmd_name.to_string(); let res = tokio::task::spawn_blocking(move || { @@ -2519,7 +2620,10 @@ impl LocalRepo { .stderr(Stdio::piped()); let mut child = cmd.spawn().map_err(GitError::Io)?; { - let stdin = child.stdin.as_mut().unwrap(); + let stdin = child + .stdin + .as_mut() + .ok_or_else(|| std::io::Error::other("git stdin unavailable"))?; stdin.write_all(&stdin_bytes).map_err(GitError::Io)?; } child.wait_with_output().map_err(GitError::Io) @@ -2559,8 +2663,14 @@ impl LocalRepo { .stdout(Stdio::piped()) .stderr(Stdio::piped()); let mut child = cmd.spawn().map_err(GitError::Io)?; - let mut stdin = child.stdin.take().unwrap(); - let mut stdout = child.stdout.take().unwrap(); + let mut stdin = child + .stdin + .take() + .ok_or_else(|| std::io::Error::other("git stdin unavailable"))?; + let mut stdout = child + .stdout + .take() + .ok_or_else(|| std::io::Error::other("git stdout unavailable"))?; // Copy the request body into stdin first, then close stdin so the // subprocess sees EOF and can finish + exit. Only then drain stdout: // `copy_out` blocks on stdout EOF (subprocess exit), and the subprocess @@ -2630,8 +2740,7 @@ fn unique_suffix() -> String { use std::time::{SystemTime, UNIX_EPOCH}; let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); + .map_or(0, |d| d.as_nanos()); format!("{}-{}", std::process::id(), nanos) } @@ -2654,12 +2763,12 @@ fn idx_object_count(idx_path: &Path) -> Result { let mut head = [0u8; 8]; f.read_exact(&mut head).map_err(GitError::Io)?; let is_v2 = &head[..4] == b"\xfftOc"; - let fanout_off = if is_v2 { 8 + 255 * 4 } else { 255 * 4 }; - f.seek(std::io::SeekFrom::Start(fanout_off as u64)) + let fanout_off: u64 = if is_v2 { 8 + 255 * 4 } else { 255 * 4 }; + f.seek(std::io::SeekFrom::Start(fanout_off)) .map_err(GitError::Io)?; let mut buf = [0u8; 4]; f.read_exact(&mut buf).map_err(GitError::Io)?; - Ok(u32::from_be_bytes(buf) as u64) + Ok(u64::from(u32::from_be_bytes(buf))) } struct IndexPackOutcome { checksum: gix_hash::ObjectId, @@ -2668,8 +2777,8 @@ struct IndexPackOutcome { object_count: u64, /// Time spent copying the pack into index-pack stdin. feed_ms: u64, - /// `exit.t_abs` from GIT_TRACE2_EVENT (whole child). index-pack itself - /// emits no region_leave events today; any that appear (future git) are + /// `exit.t_abs` from `GIT_TRACE2_EVENT` (whole child). index-pack itself + /// emits no `region_leave` events today; any that appear (future git) are /// in `phases`. git_ms: u64, /// Compact `k=ms` list: always `feed` + `git`, plus every TRACE2 @@ -2740,7 +2849,7 @@ fn git_index_pack( let mut file = file; std::io::copy(&mut file, &mut stdin).map_err(GitError::Io)?; } - let feed_ms = feed_started.elapsed().as_millis() as u64; + let feed_ms = u64::try_from(feed_started.elapsed().as_millis()).unwrap_or(u64::MAX); let output = child.wait_with_output().map_err(GitError::Io)?; let trace = std::fs::read_to_string(&trace_path).unwrap_or_default(); let _ = std::fs::remove_file(&trace_path); @@ -2797,6 +2906,11 @@ struct Trace2Phases { regions: Vec<(String, u64)>, } +#[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "Trace display milliseconds intentionally round up and saturate float-to-int conversion" +)] fn secs_to_ms(t: f64) -> u64 { let ms = (t * 1000.0).ceil() as u64; if t > 0.0 && ms == 0 { 1 } else { ms } @@ -2807,7 +2921,7 @@ fn json_str_field<'a>(line: &'a str, key: &str) -> Option<&'a str> { let pat = format!("\"{key}\":\""); let rest = line.split_once(&pat)?.1; let end = rest.find('"')?; - Some(&rest[..end]) + rest.get(..end) } fn json_f64_field(line: &str, key: &str) -> Option { @@ -2888,10 +3002,10 @@ fn find_conflict(stderr: &str) -> Option { // git update-ref prints: "cannot lock ref '' ... : ..." or similar. // Best-effort: extract a ref name appearing in a quoted context. for line in stderr.lines() { - if let Some((_, rest)) = line.split_once("cannot lock ref '") { - if let Some((name, _)) = rest.split_once('\'') { - return Some(name.to_string()); - } + if let Some((_, rest)) = line.split_once("cannot lock ref '") + && let Some((name, _)) = rest.split_once('\'') + { + return Some(name.to_string()); } if let Some((_, rest)) = line.split_once("ref ") { // "ref refs/heads/main: expected ..." @@ -2916,7 +3030,8 @@ pub(crate) fn read_refs(repo_path: &Path) -> Result { String::new() } } - Err(_) => String::new(), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(e) => return Err(e.into()), }; let mut map: BTreeMap = BTreeMap::new(); @@ -2930,10 +3045,10 @@ pub(crate) fn read_refs(repo_path: &Path) -> Result { continue; } if let Some(rest) = line.strip_prefix('^') { - if let Some(name) = &last { - if let Some((_, peeled)) = map.get_mut(name) { - *peeled = rest.trim().to_string(); - } + if let Some(name) = &last + && let Some((_, peeled)) = map.get_mut(name) + { + *peeled = rest.trim().to_string(); } continue; } @@ -2985,34 +3100,33 @@ pub(crate) fn read_refs(repo_path: &Path) -> Result { } fn walk_loose_refs(dir: &Path, prefix: &str, map: &mut BTreeMap) { - let rd = match std::fs::read_dir(dir) { - Ok(rd) => rd, - Err(_) => return, + let Ok(rd) = std::fs::read_dir(dir) else { + return; }; for ent in rd.flatten() { let path = ent.path(); let name = format!("{prefix}/{}", ent.file_name().to_string_lossy()); if path.is_dir() { walk_loose_refs(&path, &name, map); - } else if path.is_file() { - if let Ok(content) = std::fs::read_to_string(&path) { - let s = content.trim(); - if let Some(t) = s.strip_prefix("ref: ") { - // Symbolic loose ref: resolve target oid later if present. - // We record the target name in the oid slot is wrong; - // instead skip (packed-refs usually has the real value, or - // the symref target is resolved at read time elsewhere). - // For HEAD-only symref we handle separately; loose symrefs - // under refs/ are rare. Record empty oid if unresolved. - let target = t.trim(); - if let Some((o, _)) = map.get(target).cloned() { - map.insert(name, (o, String::new())); - } - continue; - } - if !s.is_empty() { - map.insert(name, (s.to_string(), String::new())); + } else if path.is_file() + && let Ok(content) = std::fs::read_to_string(&path) + { + let s = content.trim(); + if let Some(t) = s.strip_prefix("ref: ") { + // Symbolic loose ref: resolve target oid later if present. + // We record the target name in the oid slot is wrong; + // instead skip (packed-refs usually has the real value, or + // the symref target is resolved at read time elsewhere). + // For HEAD-only symref we handle separately; loose symrefs + // under refs/ are rare. Record empty oid if unresolved. + let target = t.trim(); + if let Some((o, _)) = map.get(target).cloned() { + map.insert(name, (o, String::new())); } + continue; + } + if !s.is_empty() { + map.insert(name, (s.to_string(), String::new())); } } } @@ -3032,12 +3146,11 @@ impl LocalRepo { if u.new_oid.bytes().all(|b| b == b'0') { continue; } - if let Ok(oid) = gix_hash::ObjectId::from_hex(u.new_oid.as_bytes()) { - if let Some(p) = peel_tag(&repo, oid) { - if p != oid { - u.new_peeled = p.to_hex().to_string(); - } - } + if let Ok(oid) = gix_hash::ObjectId::from_hex(u.new_oid.as_bytes()) + && let Some(p) = peel_tag(&repo, oid) + && p != oid + { + u.new_peeled = p.to_hex().to_string(); } } } @@ -3047,9 +3160,8 @@ fn peel_tag(repo: &gix::Repository, oid: gix_hash::ObjectId) -> Option o, - Err(_) => return None, + let Ok(obj) = repo.find_object(cur) else { + return None; }; if obj.kind == gix_object::Kind::Tag { let tag = gix_object::TagRef::from_bytes(&obj.data, kind).ok()?; @@ -3137,7 +3249,7 @@ fn locate_pack_offset(path: &Path) -> Option { if n == 0 { return None; } - if let Some(i) = find_subsequence(&buf[..n], b"PACK") { + if let Some(i) = find_subsequence(buf.get(..n)?, b"PACK") { return Some(pos + i as u64); } // Seek back a little to handle boundary splits. @@ -3193,7 +3305,7 @@ struct ConnectivityVisitor<'a> { missing: Option, } -impl<'a> TreeVisit for ConnectivityVisitor<'a> { +impl TreeVisit for ConnectivityVisitor<'_> { fn pop_front_tracked_path_and_set_current(&mut self) {} fn pop_back_tracked_path_and_set_current(&mut self) {} fn push_back_tracked_path_component(&mut self, _c: &gix_object::bstr::BStr) {} @@ -3225,17 +3337,15 @@ impl<'a> TreeVisit for ConnectivityVisitor<'a> { if entry.mode.is_commit() { return std::ops::ControlFlow::Continue(true); } - if self.seen.insert(entry.oid.to_owned()) { - if !self.repo.has_object(entry.oid) { - self.missing = Some(entry.oid.to_owned()); - return std::ops::ControlFlow::Break(()); - } + if self.seen.insert(entry.oid.to_owned()) && !self.repo.has_object(entry.oid) { + self.missing = Some(entry.oid.to_owned()); + return std::ops::ControlFlow::Break(()); } std::ops::ControlFlow::Continue(true) } } -/// Parse a filter spec string into a PackFilter. +/// Parse a filter spec string into a `PackFilter`. #[derive(Debug, Clone)] pub(crate) enum PackFilter { None, @@ -3248,21 +3358,21 @@ pub(crate) fn parse_filter(spec: &str) -> PackFilter { if spec == "blob:none" { return PackFilter::BlobNone; } - if let Some(rest) = spec.strip_prefix("blob:limit=") { - if let Ok(n) = rest.parse::() { - return PackFilter::BlobLimit(n); - } + if let Some(rest) = spec.strip_prefix("blob:limit=") + && let Ok(n) = rest.parse::() + { + return PackFilter::BlobLimit(n); } - if let Some(rest) = spec.strip_prefix("tree:") { - if let Ok(n) = rest.parse::() { - return PackFilter::Tree(n); - } + if let Some(rest) = spec.strip_prefix("tree:") + && let Ok(n) = rest.parse::() + { + return PackFilter::Tree(n); } PackFilter::None } /// Compute the object set for a pack: reachable(wants) - reachable(common -/// haves), honoring filters and include_tag. +/// haves), honoring filters and `include_tag`. /// /// Handles non-commit wants (blobs, trees, tags) for partial-clone lazy fetch /// where the client sends `want ` directly. @@ -3274,7 +3384,7 @@ pub(crate) fn compute_object_set( include_tag: bool, deepen: Option, ) -> Result, GitError> { - let pack_filter = filter.map(parse_filter).unwrap_or(PackFilter::None); + let pack_filter = filter.map_or(PackFilter::None, parse_filter); let mut set: HashSet = HashSet::new(); let mut buf = Vec::new(); let kind = repo.object_hash(); @@ -3292,21 +3402,20 @@ pub(crate) fn compute_object_set( // Follow tag chain to final target. let mut cur = *w; loop { - let obj = repo.find_object(cur).map_err(|e| ge(e))?; + let obj = repo.find_object(cur).map_err(ge)?; if obj.kind != ObjKind::Tag { break; } - let tag = - gix_object::TagRef::from_bytes(&obj.data, kind).map_err(|e| ge(e))?; + let tag = gix_object::TagRef::from_bytes(&obj.data, kind).map_err(ge)?; let target = tag.target(); set.insert(target); cur = target; } // If the final target is a commit, rev-walk from it. - if let Ok(Some(h)) = repo.objects.try_header(&cur) { - if h.kind == ObjKind::Commit { - commit_wants.push(cur); - } + if let Ok(Some(h)) = repo.objects.try_header(&cur) + && h.kind == ObjKind::Commit + { + commit_wants.push(cur); } } ObjKind::Tree => { @@ -3339,10 +3448,10 @@ pub(crate) fn compute_object_set( .rev_walk(commit_wants.iter().copied()) .with_hidden(hidden.iter().copied()) .all() - .map_err(|e| ge(e))?; + .map_err(ge)?; for item in walk { - let info = item.map_err(|e| ge(e))?; + let info = item.map_err(ge)?; let cid = info.id; if !set.insert(cid) { continue; @@ -3353,11 +3462,8 @@ pub(crate) fn compute_object_set( // commit and walking its tree here as well. continue; } - let mut commit = repo - .objects - .find_commit_iter(&cid, &mut buf) - .map_err(|e| ge(e))?; - let tree_id = commit.tree_id().map_err(|e| ge(e))?; + let mut commit = repo.objects.find_commit_iter(&cid, &mut buf).map_err(ge)?; + let tree_id = commit.tree_id().map_err(ge)?; // `tree:0` sends no trees at all, the root included. if matches!(pack_filter, PackFilter::Tree(0)) { @@ -3378,14 +3484,13 @@ pub(crate) fn compute_object_set( if set.contains(&tag_oid) { continue; } - if let Ok(obj) = repo.find_object(tag_oid) { - if obj.kind == ObjKind::Tag { - if let Ok(tag) = gix_object::TagRef::from_bytes(&obj.data, kind) { - let target = tag.target(); - if set.contains(&target) { - set.insert(tag_oid); - } - } + if let Ok(obj) = repo.find_object(tag_oid) + && obj.kind == ObjKind::Tag + && let Ok(tag) = gix_object::TagRef::from_bytes(&obj.data, kind) + { + let target = tag.target(); + if set.contains(&target) { + set.insert(tag_oid); } } } @@ -3408,23 +3513,20 @@ pub(crate) fn walk_tree_with_filter( buf: &mut Vec, ) -> Result<(), GitError> { // Collect entries first to end the mutable borrow of buf before recursing. - let tree_iter = repo - .objects - .find_tree_iter(&tree_id, buf) - .map_err(|e| ge(e))?; + let tree_iter = repo.objects.find_tree_iter(&tree_id, buf).map_err(ge)?; let entries: Vec<(gix_object::tree::EntryMode, gix_hash::ObjectId)> = tree_iter .map(|res| { - let e = res.map_err(|e| ge(e))?; + let e = res.map_err(ge)?; Ok((e.mode, e.oid.to_owned())) }) .collect::>()?; for (mode, oid) in entries { if mode.is_tree() { - if let PackFilter::Tree(max_depth) = filter { - if depth + 1 > *max_depth { - set.insert(oid); - continue; - } + if let PackFilter::Tree(max_depth) = filter + && depth + 1 > *max_depth + { + set.insert(oid); + continue; } if set.insert(oid) { walk_tree_with_filter(repo, oid, set, filter, depth + 1, buf)?; @@ -3437,12 +3539,11 @@ pub(crate) fn walk_tree_with_filter( if matches!(filter, PackFilter::BlobNone) { continue; } - if let PackFilter::BlobLimit(limit) = filter { - if let Ok(Some(hdr)) = repo.objects.try_header(&oid) { - if hdr.size > *limit { - continue; - } - } + if let PackFilter::BlobLimit(limit) = filter + && let Ok(Some(hdr)) = repo.objects.try_header(&oid) + && hdr.size > *limit + { + continue; } set.insert(oid); } @@ -3479,12 +3580,8 @@ pub(crate) fn compute_shallow( if !seen.insert(*cid) { continue; } - let commit = repo - .objects - .find_commit_iter(cid, &mut buf) - .map_err(|e| ge(e))?; - let parents: Vec = - commit.parent_ids().map(|p| p.to_owned()).collect(); + let commit = repo.objects.find_commit_iter(cid, &mut buf).map_err(ge)?; + let parents: Vec = commit.parent_ids().collect(); if d == depth.max(1) { if !parents.is_empty() { shallow.push(*cid); @@ -3504,12 +3601,14 @@ pub(crate) fn compute_shallow( } /// Compute the SHA checksum trailer for a pack header (used for empty packs). -pub(crate) fn compute_pack_trailer(data: &[u8], kind: gix_hash::Kind) -> gix_hash::ObjectId { +pub(crate) fn compute_pack_trailer( + data: &[u8], + kind: gix_hash::Kind, +) -> Result { use gix_hash::hasher; let mut h = hasher(kind); h.update(data); - // try_finalize always succeeds when the hasher has been fed data. - h.try_finalize().expect("hash finalization must succeed") + h.try_finalize().map_err(ge) } /// The hash git names a split commit-graph layer by: the file's trailing @@ -3517,20 +3616,22 @@ pub(crate) fn compute_pack_trailer(data: &[u8], kind: gix_hash::Kind) -> gix_has fn commit_graph_layer_hash(path: &Path) -> Result { let data = std::fs::read(path).map_err(GitError::Io)?; // Header: "CGPH" version(1) hash-version(1) chunks(1) base-graphs(1) - if data.len() < 8 || &data[..4] != b"CGPH" { + if data.len() < 8 || !data.starts_with(b"CGPH") { return Err(GitError::InvalidInput(format!( "{} is not a commit-graph", path.display() ))); } - let len = if data[5] == 2 { 32 } else { 20 }; + let len = if data.get(5) == Some(&2) { 32 } else { 20 }; if data.len() < 8 + len { return Err(GitError::InvalidInput(format!( "{} is truncated", path.display() ))); } - Ok(hex::encode(&data[data.len() - len..])) + Ok(hex::encode(data.get(data.len() - len..).ok_or_else( + || GitError::InvalidInput("truncated graph checksum".into()), + )?)) } /// Derive a pack's reverse index (`.rev`, RIDX v1) from its `.idx`: header @@ -3550,8 +3651,8 @@ pub fn write_rev_from_idx( let n = index.num_objects(); let mut by_offset: Vec<(u64, u32)> = index .iter() - .enumerate() - .map(|(i, e)| (e.pack_offset, i as u32)) + .zip(0..n) + .map(|(e, i)| (e.pack_offset, i)) .collect(); by_offset.sort_unstable(); let mut out = Vec::with_capacity(12 + 4 * n as usize + 2 * kind.len_in_bytes()); @@ -3644,8 +3745,8 @@ mod index_pack_trace_tests { .spawn() .unwrap(); { - let mut stdin = child.stdin.take().unwrap(); use std::io::Write; + let mut stdin = child.stdin.take().expect("piped stdin"); stdin.write_all(b"HEAD\n").unwrap(); } let out = child.wait_with_output().unwrap(); diff --git a/crates/walgit-git/src/pkt.rs b/crates/walgit-git/src/pkt.rs index a5c7dee..5db7944 100644 --- a/crates/walgit-git/src/pkt.rs +++ b/crates/walgit-git/src/pkt.rs @@ -78,7 +78,7 @@ pub async fn read_pkt_line(r: &mut R) -> Result return Ok(Some(PktLine::Flush)), 1 => return Ok(Some(PktLine::Delim)), @@ -101,19 +101,23 @@ async fn read_exact_or_eof( buf: &mut [u8], ) -> Result { let mut filled = 0; - while filled < buf.len() { - let n = r.read(&mut buf[filled..]).await.map_err(io_to_git)?; + let mut remaining = buf; + while !remaining.is_empty() { + let n = r.read(remaining).await.map_err(io_to_git)?; if n == 0 { break; } + remaining = remaining + .get_mut(n..) + .ok_or_else(|| GitError::Protocol("reader exceeded buffer".into()))?; filled += n; } Ok(filled) } -fn parse_pkt_len(hdr: &[u8; 4]) -> Result { +fn parse_pkt_len(hdr: [u8; 4]) -> Result { let mut val = 0usize; - for &b in hdr { + for b in hdr { let d = match b { b'0'..=b'9' => b - b'0', b'a'..=b'f' => b - b'a' + 10, @@ -136,17 +140,12 @@ pub async fn write_pkt_line(w: &mut W, data: &[u8]) -> Re w.write_all(b"0004").await.map_err(io_to_git)?; return Ok(()); } - let mut off = 0; - while off < data.len() { - let chunk = (data.len() - off).min(MAX_PKT_DATA); - let total = chunk + 4; + for chunk in data.chunks(MAX_PKT_DATA) { + let total = chunk.len() + 4; w.write_all(pkt_len_hex(total).as_bytes()) .await .map_err(io_to_git)?; - w.write_all(&data[off..off + chunk]) - .await - .map_err(io_to_git)?; - off += chunk; + w.write_all(chunk).await.map_err(io_to_git)?; } Ok(()) } @@ -212,20 +211,14 @@ impl Sideband { self.w.write_all(&[channel]).await.map_err(io_to_git)?; return Ok(()); } - let mut off = 0; - while off < buf.len() { - let chunk = (buf.len() - off).min(MAX); - let total = chunk + 4 + 1; + for chunk in buf.chunks(MAX) { + let total = chunk.len() + 4 + 1; self.w .write_all(pkt_len_hex(total).as_bytes()) .await .map_err(io_to_git)?; self.w.write_all(&[channel]).await.map_err(io_to_git)?; - self.w - .write_all(&buf[off..off + chunk]) - .await - .map_err(io_to_git)?; - off += chunk; + self.w.write_all(chunk).await.map_err(io_to_git)?; } Ok(()) } @@ -251,7 +244,7 @@ pub struct V2Command { impl V2Command { pub fn cap(&self, key: &str) -> Option<&str> { - self.caps.get(key).map(|s| s.as_str()) + self.caps.get(key).map(std::string::String::as_str) } pub fn has_cap(&self, key: &str) -> bool { self.caps.contains_key(key) @@ -352,7 +345,7 @@ pub async fn read_ls_refs_args( let line = String::from_utf8_lossy(&data); parse_ls_refs_line(&mut req, line.trim_end()); } - Some(PktLine::Delim) => continue, + Some(PktLine::Delim) => {} Some(PktLine::Flush | PktLine::ResponseEnd) | None => break, } } @@ -384,9 +377,13 @@ fn io_to_git(e: std::io::Error) -> GitError { /// Encode a literal data pkt-line into a buffer (sync helper for building /// advertisement/section bytes). +#[expect( + clippy::indexing_slicing, + reason = "Each index is masked to 0..16 for the 16-byte hex table" +)] pub fn encode_data(buf: &mut Vec, data: &[u8]) { - let total = data.len() + 4; const HEX: &[u8; 16] = b"0123456789abcdef"; + let total = data.len() + 4; buf.extend_from_slice(&[ HEX[(total >> 12) & 0xf], HEX[(total >> 8) & 0xf], diff --git a/crates/walgit-git/src/receive.rs b/crates/walgit-git/src/receive.rs index 8b57a6a..d8f2851 100644 --- a/crates/walgit-git/src/receive.rs +++ b/crates/walgit-git/src/receive.rs @@ -18,6 +18,10 @@ use crate::{GitError, RefSnapshotData}; /// Capabilities negotiated by the client in the first receive-pack command. #[derive(Debug, Default, Clone)] +#[expect( + clippy::struct_excessive_bools, + reason = "Git capabilities are independent protocol flags" +)] pub struct ReceiveCaps { pub report_status: bool, pub report_status_v2: bool, @@ -63,8 +67,8 @@ impl AsyncRead for PrefixedReader { let this = self.get_mut(); if !this.prefix.is_empty() { let n = this.prefix.len().min(buf.remaining()); - for _ in 0..n { - buf.put_slice(&[this.prefix.pop_front().unwrap()]); + for byte in this.prefix.drain(..n) { + buf.put_slice(&[byte]); } return std::task::Poll::Ready(Ok(())); } @@ -99,8 +103,14 @@ pub async fn parse( let first = loop { match pkt::read_pkt_line(&mut r).await? { Some(PktLine::Data(b)) if b.starts_with(b"shallow ") => { - caps.shallow - .push(String::from_utf8_lossy(&b[8..]).trim().to_string()); + caps.shallow.push( + String::from_utf8_lossy( + b.strip_prefix(b"shallow ") + .ok_or_else(|| GitError::Protocol("missing shallow prefix".into()))?, + ) + .trim() + .to_string(), + ); } other => break other, } @@ -115,7 +125,7 @@ pub async fn parse( }; return Ok((txn, caps, PrefixedReader::new(Vec::new(), r))); } - Some(PktLine::Delim) | Some(PktLine::ResponseEnd) => { + Some(PktLine::Delim | PktLine::ResponseEnd) => { return Err(GitError::Protocol( "unexpected delim before commands".into(), )); @@ -134,11 +144,16 @@ pub async fn parse( loop { let line = pkt::read_pkt_line(&mut r).await?; match line { - None | Some(PktLine::Flush) => break, - Some(PktLine::Delim) | Some(PktLine::ResponseEnd) => break, + None | Some(PktLine::Flush | PktLine::Delim | PktLine::ResponseEnd) => break, Some(PktLine::Data(b)) if b.starts_with(b"shallow ") => { - caps.shallow - .push(String::from_utf8_lossy(&b[8..]).trim().to_string()); + caps.shallow.push( + String::from_utf8_lossy( + b.strip_prefix(b"shallow ") + .ok_or_else(|| GitError::Protocol("missing shallow prefix".into()))?, + ) + .trim() + .to_string(), + ); } Some(PktLine::Data(b)) => { let (update, _) = parse_command_line(&b)?; @@ -152,8 +167,7 @@ pub async fn parse( loop { let line = pkt::read_pkt_line(&mut r).await?; match line { - None | Some(PktLine::Flush) => break, - Some(PktLine::Delim) | Some(PktLine::ResponseEnd) => break, + None | Some(PktLine::Flush | PktLine::Delim | PktLine::ResponseEnd) => break, Some(PktLine::Data(b)) => { push_options.push( String::from_utf8_lossy(&b) @@ -175,10 +189,9 @@ pub async fn parse( fn parse_command_line(b: &[u8]) -> Result<(walgit_proto::v1::RefUpdate, String), GitError> { // First line: " \0". Subsequent lines have no caps. - let (cmd_bytes, caps_bytes) = match b.iter().position(|&c| c == 0) { - Some(idx) => (&b[..idx], &b[idx + 1..]), - None => (b, &b[..0]), - }; + let mut sections = b.splitn(2, |&c| c == 0); + let cmd_bytes = sections.next().unwrap_or_default(); + let caps_bytes = sections.next().unwrap_or_default(); let s = String::from_utf8_lossy(cmd_bytes); let s = s.trim_end_matches('\n'); let mut parts = s.splitn(3, ' '); @@ -203,7 +216,7 @@ fn parse_command_line(b: &[u8]) -> Result<(walgit_proto::v1::RefUpdate, String), } fn apply_caps(caps: &mut ReceiveCaps, s: &str) { - for tok in s.split(|c: char| c == ' ' || c == '\n') { + for tok in s.split([' ', '\n']) { let tok = tok.trim(); if tok.is_empty() { continue; @@ -216,11 +229,11 @@ fn apply_caps(caps: &mut ReceiveCaps, s: &str) { "quiet" => caps.quiet = true, "push-options" => caps.push_options = true, "ofs-delta" => caps.ofs_delta = true, - _ if tok.starts_with("agent=") => caps.agent = Some(tok[6..].to_string()), - _ if tok.starts_with("object-format=") => { - caps.object_format = Some(tok[14..].to_string()) - } - _ => {} + _ => match tok.split_once('=') { + Some(("agent", value)) => caps.agent = Some(value.to_string()), + Some(("object-format", value)) => caps.object_format = Some(value.to_string()), + _ => {} + }, } } } @@ -298,7 +311,7 @@ pub async fn report_status( /// Convenience: build a [`RefTransaction`] from a ref snapshot diff is not /// provided; callers construct transactions directly. This helper converts a -/// [`RefSnapshotData`] into a transaction that creates all refs (old_oid = +/// [`RefSnapshotData`] into a transaction that creates all refs (`old_oid` = /// zero), useful for materializing a checkpoint. pub fn txn_from_snapshot(snap: &RefSnapshotData) -> walgit_proto::v1::RefTransaction { let mut updates: Vec = snap diff --git a/crates/walgit-git/src/repair.rs b/crates/walgit-git/src/repair.rs index 9888734..a8fb562 100644 --- a/crates/walgit-git/src/repair.rs +++ b/crates/walgit-git/src/repair.rs @@ -110,7 +110,9 @@ pub async fn fetch_objects_as_pack( .map_err(GitError::Io)?; { use tokio::io::AsyncWriteExt; - let mut stdin = child.stdin.take().expect("stdin"); + let mut stdin = child.stdin.take().ok_or_else(|| { + GitError::InvalidInput("git pack-objects stdin unavailable".to_owned()) + })?; let mut input = oids.join("\n"); input.push('\n'); stdin @@ -141,7 +143,7 @@ pub async fn fetch_objects_as_pack( let mut first_missing = None; for o in oids { match gix_hash::ObjectId::from_hex(o.as_bytes()) { - Ok(id) if index.lookup(&id).is_some() => objects += 1, + Ok(id) if index.lookup(id).is_some() => objects += 1, _ => { first_missing.get_or_insert(o.as_str()); } diff --git a/crates/walgit-git/src/upload_gix.rs b/crates/walgit-git/src/upload_gix.rs index e5fffed..3171854 100644 --- a/crates/walgit-git/src/upload_gix.rs +++ b/crates/walgit-git/src/upload_gix.rs @@ -72,7 +72,7 @@ fn blocking_section(f: impl FnOnce() -> T) -> T { } /// Write one section line, wrapped in a band-1 frame when the client asked -/// for `sideband-all` (flush/delim stay raw, as in git's packet_writer). +/// for `sideband-all` (flush/delim stay raw, as in git's `packet_writer`). fn line(buf: &mut Vec, data: &[u8], sideband_all: bool) { if sideband_all { let mut framed = Vec::with_capacity(data.len() + 1); @@ -207,10 +207,17 @@ impl LocalRepo { ) -> Result { let mut header = String::from("# v2 git bundle\n"); for p in prerequisites { - header.push_str(&format!("-{} \n", p.to_hex())); + { + let _ = std::fmt::Write::write_fmt(&mut header, format_args!("-{} \n", p.to_hex())); + }; } for (name, oid) in refs { - header.push_str(&format!("{} {name}\n", oid.to_hex())); + { + let _ = std::fmt::Write::write_fmt( + &mut header, + format_args!("{} {name}\n", oid.to_hex()), + ); + }; } header.push('\n'); out.write_all(header.as_bytes()) @@ -271,7 +278,13 @@ impl LocalRepo { let found = f.fault(&missing).await?; if found < missing.len() { return Err(GitError::MissingObject { - oid: missing[0].to_hex().to_string(), + oid: missing + .first() + .ok_or_else(|| { + GitError::InvalidInput("empty missing-object set".into()) + })? + .to_hex() + .to_string(), }); } self.refresh_async().await?; @@ -279,11 +292,7 @@ impl LocalRepo { } // ---- enumerate (sync, retried after faulting missing objects) ---- - let filter = req - .filter - .as_deref() - .map(parse_filter) - .unwrap_or(PackFilter::None); + let filter = req.filter.as_deref().map_or(PackFilter::None, parse_filter); let has_filter = req.filter.is_some(); let mut rounds = 0usize; let (set, commits, diffed) = loop { @@ -291,7 +300,7 @@ impl LocalRepo { // seconds on big ranges: never on an async worker (D19). let attempt = blocking_section(|| { let repo = self.gix(); - enumerate(&repo, &req, &common_haves, &filter, faulter) + enumerate(&repo, req, common_haves, &filter, faulter) })?; match attempt { Enumerated::Done { @@ -303,7 +312,13 @@ impl LocalRepo { rounds += 1; let Some(f) = faulter else { return Err(GitError::MissingObject { - oid: missing[0].to_hex().to_string(), + oid: missing + .first() + .ok_or_else(|| { + GitError::InvalidInput("empty missing-object set".into()) + })? + .to_hex() + .to_string(), }); }; if rounds > MAX_FAULT_ROUNDS { @@ -320,7 +335,13 @@ impl LocalRepo { let found = f.fault(&missing).await?; if found == 0 { return Err(GitError::MissingObject { - oid: missing[0].to_hex().to_string(), + oid: missing + .first() + .ok_or_else(|| { + GitError::InvalidInput("empty missing-object set".into()) + })? + .to_hex() + .to_string(), }); } self.refresh_async().await?; @@ -354,7 +375,13 @@ impl LocalRepo { if !missing.is_empty() { let Some(f) = faulter else { return Err(GitError::MissingObject { - oid: missing[0].to_hex().to_string(), + oid: missing + .first() + .ok_or_else(|| { + GitError::InvalidInput("empty missing-object set".into()) + })? + .to_hex() + .to_string(), }); }; sink.progress(&format!( @@ -365,7 +392,13 @@ impl LocalRepo { let found = f.fault(&missing).await?; if found < missing.len() { return Err(GitError::MissingObject { - oid: missing[0].to_hex().to_string(), + oid: missing + .first() + .ok_or_else(|| { + GitError::InvalidInput("empty missing-object set".into()) + })? + .to_hex() + .to_string(), }); } self.refresh_async().await?; @@ -419,8 +452,8 @@ impl LocalRepo { .await .map_err(|e| GitError::Protocol(format!("pack generator panicked: {e}")))??; tracing::debug!( - enumerate_ms = t_enum.as_millis() as u64, - total_ms = t_start.elapsed().as_millis() as u64, + enumerate_ms = u64::try_from(t_enum.as_millis()).unwrap_or(u64::MAX), + total_ms = u64::try_from(t_start.elapsed().as_millis()).unwrap_or(u64::MAX), objects = num_objects, bytes, rounds, @@ -429,7 +462,7 @@ impl LocalRepo { sink.progress(&format!("Total {num_objects} objects, {bytes} bytes\n")) .await; Ok(UploadPackStats { - objects: num_objects as u64, + objects: u64::from(num_objects), bytes, }) } @@ -456,7 +489,7 @@ impl PackOut { } PackOut::Sideband { .. } => {} PackOut::Raw(_) => { - tracing::debug!(target: "walgit_git::upload_gix", "{}", text.trim_end()) + tracing::debug!(target: "walgit_git::upload_gix", "{}", text.trim_end()); } } } @@ -494,7 +527,7 @@ struct ChanWriter { impl ChanWriter { fn flush_all(&mut self) -> Result<(), GitError> { if !self.buf.is_empty() { - let chunk = std::mem::replace(&mut self.buf, Vec::new()); + let chunk = std::mem::take(&mut self.buf); self.tx.blocking_send(chunk).map_err(|_| { GitError::Io(std::io::Error::new( std::io::ErrorKind::BrokenPipe, @@ -568,7 +601,9 @@ fn generate_pack_streaming( let thread_limit = if small { Some(1) } else { - std::thread::available_parallelism().map(|n| n.get()).ok() + std::thread::available_parallelism() + .map(std::num::NonZero::get) + .ok() }; let chunk_size = if small { 64 } else { 256 }; let interrupt = std::sync::atomic::AtomicBool::new(false); @@ -587,12 +622,13 @@ fn generate_pack_streaming( if counts.is_empty() { let header = gix_pack::data::header::encode(PackVersion::V2, 0); let mut buf = header.to_vec(); - let trailer = crate::compute_pack_trailer(&buf, object_hash); + let trailer = crate::compute_pack_trailer(&buf, object_hash)?; buf.extend_from_slice(trailer.as_slice()); out.write_all(&buf).map_err(GitError::Io)?; return Ok(0); } - let num_entries = counts.len() as u32; + let num_entries = u32::try_from(counts.len()) + .map_err(|_| GitError::InvalidInput("pack exceeds u32 object count".into()))?; let progress: Box = Box::new(gix_features::progress::Discard); let entries = entry::iter_from_counts( @@ -609,14 +645,14 @@ fn generate_pack_streaming( }, ); let entries_in_order = gix_features::parallel::InOrderIter::from(entries); - let mut pack_iter = FromEntriesIter::new( + let pack_iter = FromEntriesIter::new( entries_in_order, out, num_entries, PackVersion::V2, object_hash, ); - while let Some(result) = pack_iter.next() { + for result in pack_iter { result.map_err(ge)?; } Ok(num_entries) @@ -793,19 +829,16 @@ fn enumerate( let mut parent_trees: Vec = Vec::with_capacity(parent_ids.len()); let mut deferred = false; for p in &parent_ids { - match repo.objects.try_find(p, &mut buf).map_err(GitError::Gix)? { - Some(obj) => { - match gix_object::CommitRefIter::from_bytes(obj.data, kind).tree_id() { - Ok(t) => parent_trees.push(Old::Tree(t)), - Err(e) => return Err(ge(e)), - } - } - None => { - // Parent commit not local (base): fault it, diff this - // commit on the retry. - missing.push(*p); - deferred = true; + if let Some(obj) = repo.objects.try_find(p, &mut buf).map_err(GitError::Gix)? { + match gix_object::CommitRefIter::from_bytes(obj.data, kind).tree_id() { + Ok(t) => parent_trees.push(Old::Tree(t)), + Err(e) => return Err(ge(e)), } + } else { + // Parent commit not local (base): fault it, diff this + // commit on the retry. + missing.push(*p); + deferred = true; } } if deferred { @@ -835,24 +868,22 @@ fn enumerate( } // include-tag: annotated tags whose target is in the set. - if req.include_tag { - if let Ok(snap) = crate::read_refs(repo.path()) { - for r in &snap.refs { - let Ok(tag_oid) = gix_hash::ObjectId::from_hex(r.oid.as_bytes()) else { - continue; - }; - if set.contains(&tag_oid) { - continue; - } - if let Ok(Some(obj)) = repo.objects.try_find(&tag_oid, &mut buf) { - if obj.kind == ObjKind::Tag { - if let Ok(tag) = gix_object::TagRef::from_bytes(obj.data, kind) { - if set.contains(&tag.target()) { - set.insert(tag_oid); - } - } - } - } + if req.include_tag + && let Ok(snap) = crate::read_refs(repo.path()) + { + for r in &snap.refs { + let Ok(tag_oid) = gix_hash::ObjectId::from_hex(r.oid.as_bytes()) else { + continue; + }; + if set.contains(&tag_oid) { + continue; + } + if let Ok(Some(obj)) = repo.objects.try_find(&tag_oid, &mut buf) + && obj.kind == ObjKind::Tag + && let Ok(tag) = gix_object::TagRef::from_bytes(obj.data, kind) + && set.contains(&tag.target()) + { + set.insert(tag_oid); } } } @@ -923,15 +954,14 @@ fn diff_tree_new_objects( for o in olds { match o { Old::Absent => old_maps.push(HashMap::new()), - Old::Tree(oid) => match tree_entries(repo, oid, buf)? { - Some(entries) => { - old_maps.push(entries.into_iter().map(|(m, n, o)| (n, (m, o))).collect()) - } - None => { + Old::Tree(oid) => { + if let Some(entries) = tree_entries(repo, oid, buf)? { + old_maps.push(entries.into_iter().map(|(m, n, o)| (n, (m, o))).collect()); + } else { missing.push(*oid); deferred = true; } - }, + } } } if deferred { @@ -953,11 +983,11 @@ fn diff_tree_new_objects( continue; } if mode.is_tree() { - if let PackFilter::Tree(max) = filter { - if depth + 1 > *max { - set.insert(oid); - continue; - } + if let PackFilter::Tree(max) = filter + && depth + 1 > *max + { + set.insert(oid); + continue; } if set.insert(oid) { let sub_olds: Vec = old_maps @@ -973,10 +1003,10 @@ fn diff_tree_new_objects( match filter { PackFilter::BlobNone => continue, PackFilter::BlobLimit(limit) => { - if let Ok(Some(h)) = repo.objects.try_header(&oid) { - if h.size > *limit { - continue; - } + if let Ok(Some(h)) = repo.objects.try_header(&oid) + && h.size > *limit + { + continue; } } _ => {} @@ -1022,8 +1052,12 @@ mod frozen_source_tests { // Two packs with distinct content, newest first in gix's load order. let mut blobs = Vec::new(); for (i, words) in ["one pack", "two pack"].iter().enumerate() { + use std::io::Write; + let content = format!("{words} {}\n", "x".repeat(300 + i * 50)); let oid = { + use std::io::Write; + let mut c = std::process::Command::new("git") .arg("-C") .arg(dir) @@ -1032,7 +1066,7 @@ mod frozen_source_tests { .stdout(std::process::Stdio::piped()) .spawn() .unwrap(); - use std::io::Write; + c.stdin .take() .unwrap() @@ -1052,7 +1086,7 @@ mod frozen_source_tests { .stdout(std::process::Stdio::piped()) .spawn() .unwrap(); - use std::io::Write; + c.stdin .take() .unwrap() @@ -1111,8 +1145,12 @@ mod frozen_source_tests { // slot (same path). Keep adding packs until that happens. let mut shifted = false; for i in 0..24 { + use std::io::Write; + let content = format!("later pack {i} {}\n", "y".repeat(200 + i)); let oid = { + use std::io::Write; + let mut c = std::process::Command::new("git") .arg("-C") .arg(dir) @@ -1121,7 +1159,7 @@ mod frozen_source_tests { .stdout(std::process::Stdio::piped()) .spawn() .unwrap(); - use std::io::Write; + c.stdin .take() .unwrap() @@ -1140,7 +1178,7 @@ mod frozen_source_tests { .stdout(std::process::Stdio::piped()) .spawn() .unwrap(); - use std::io::Write; + c.stdin .take() .unwrap() diff --git a/crates/walgit-git/tests/commit_graph.rs b/crates/walgit-git/tests/commit_graph.rs index 9344717..fa358fd 100644 --- a/crates/walgit-git/tests/commit_graph.rs +++ b/crates/walgit-git/tests/commit_graph.rs @@ -1,3 +1,6 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::unwrap_used, clippy::many_single_char_names)] + mod common; use std::process::Command; diff --git a/crates/walgit-git/tests/common/mod.rs b/crates/walgit-git/tests/common/mod.rs index b22926b..2032fad 100644 --- a/crates/walgit-git/tests/common/mod.rs +++ b/crates/walgit-git/tests/common/mod.rs @@ -1,3 +1,11 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow( + clippy::expect_used, + clippy::indexing_slicing, + clippy::panic, + clippy::unwrap_used +)] + //! Shared helpers for walgit-git integration tests: build synthetic repos with //! upstream `git` and produce packs via `git pack-objects`. Each test binary //! uses a subset, so unused-item warnings are expected here. @@ -14,7 +22,7 @@ pub struct SourceRepo { _tmp: TempDir, } -/// Owned cursor satisfying `AsyncRead + Unpin + Send + 'static` (ingest_pack +/// Owned cursor satisfying `AsyncRead + Unpin + Send + 'static` (`ingest_pack` /// requires `'static`, so a borrowed `&[u8]` won't do). pub fn cursor(b: Vec) -> std::io::Cursor> { std::io::Cursor::new(b) @@ -162,14 +170,13 @@ pub fn run_git(dir: &std::path::Path, args: &[&str]) -> String { .current_dir(dir) .args(args) .output() - .unwrap_or_else(|e| panic!("git {:?}: {e}", args)); - if !out.status.success() { - panic!( - "git {:?} failed: {}", - args, - String::from_utf8_lossy(&out.stderr) - ); - } + .unwrap_or_else(|e| panic!("git {args:?}: {e}")); + assert!( + out.status.success(), + "git {:?} failed: {}", + args, + String::from_utf8_lossy(&out.stderr) + ); String::from_utf8_lossy(&out.stdout).into_owned() } @@ -249,7 +256,7 @@ pub fn extract_packfile(response: &[u8]) -> Vec { } else { &b[..] }; - if line.strip_suffix(b"\n").map_or(false, |s| s == b"packfile") { + if line.strip_suffix(b"\n").is_some_and(|s| s == b"packfile") { in_packfile = true; } continue; @@ -284,7 +291,7 @@ pub fn has_nak(response: &[u8]) -> bool { } /// Count object types in a pack file via `git verify-pack -v` in a fresh bare -/// repo. Returns (num_blobs, num_commits, num_trees, num_tags). +/// repo. Returns (`num_blobs`, `num_commits`, `num_trees`, `num_tags`). pub fn pack_object_types(pack: &[u8]) -> (u64, u64, u64, u64) { let tmp = fresh_bare(); // Write the pack and index it. diff --git a/crates/walgit-git/tests/connectivity.rs b/crates/walgit-git/tests/connectivity.rs index 8f73e90..bc2d93a 100644 --- a/crates/walgit-git/tests/connectivity.rs +++ b/crates/walgit-git/tests/connectivity.rs @@ -1,3 +1,6 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::unwrap_used)] + mod common; use walgit_git::{GitError, IngestOptions, LocalRepo, ObjectFormat, RepoId, gix_hash}; diff --git a/crates/walgit-git/tests/ingest.rs b/crates/walgit-git/tests/ingest.rs index cbd434e..14e1d2c 100644 --- a/crates/walgit-git/tests/ingest.rs +++ b/crates/walgit-git/tests/ingest.rs @@ -1,3 +1,5 @@ +#![allow(clippy::format_collect)] + mod common; use std::path::Path; @@ -41,7 +43,7 @@ async fn ingest_pack_objects_present_fsck_ok() { assert!(ingested.object_count > 0); assert_eq!( ingested.pack_path.file_name().unwrap().to_string_lossy(), - format!("pack-{}.pack", checksum) + format!("pack-{checksum}.pack") ); // Objects present. @@ -232,17 +234,34 @@ async fn ingest_large_delta_pack() { ); let mut stream = String::new(); for i in 1..=2000 { - stream.push_str(&format!( - "commit refs/heads/main\nmark :{i}\nauthor bench {i} +0000\ncommitter bench {i} +0000\n" - )); + { + let _ = std::fmt::Write::write_fmt( + &mut stream, + format_args!( + "commit refs/heads/main\nmark :{i}\nauthor bench {i} +0000\ncommitter bench {i} +0000\n" + ), + ); + }; let message = format!("commit {i}\n"); - stream.push_str(&format!("data {}\n{}\n", message.len(), message)); + { + let _ = std::fmt::Write::write_fmt( + &mut stream, + format_args!("data {}\n{}\n", message.len(), message), + ); + }; if i > 1 { - stream.push_str(&format!("from :{}\n", i - 1)); + { + let _ = std::fmt::Write::write_fmt(&mut stream, format_args!("from :{}\n", i - 1)); + }; } stream.push_str("M 100644 inline file.txt\n"); let content = format!("content {i} {}\n", "x".repeat(256)); - stream.push_str(&format!("data {}\n{}\n", content.len(), content)); + { + let _ = std::fmt::Write::write_fmt( + &mut stream, + format_args!("data {}\n{}\n", content.len(), content), + ); + }; } let mut fast_import = Command::new("git") .current_dir(source.path()) @@ -276,7 +295,7 @@ async fn ingest_large_delta_pack() { out.stdout }; assert!(full.starts_with(b"PACK")); - let expected_count = u32::from_be_bytes(full[8..12].try_into().unwrap()) as u64; + let expected_count = u64::from(u32::from_be_bytes(full[8..12].try_into().unwrap())); let full_root = tempfile::TempDir::new().unwrap(); let full_repo = LocalRepo::init( @@ -409,15 +428,14 @@ async fn ingest_failures_name_the_cause_and_leave_nothing_behind() { max_bytes, thin, }; - let pack_count = || repo.packs().map(|p| p.len()).unwrap_or(0); + let pack_count = || repo.packs().map_or(0, |p| p.len()); // 1. Oversize: refused while streaming, before index-pack ever runs. let full = src.pack(&[b.as_str()], &[], false); let err = repo .ingest_pack(cm::cursor(full.clone()), opts(false, Some(64))) .await - .err() - .expect("too big"); + .expect_err("too big"); assert!(err.to_string().contains("max_bytes 64"), "{err}"); assert_eq!(pack_count(), 0); @@ -428,8 +446,7 @@ async fn ingest_failures_name_the_cause_and_leave_nothing_behind() { let err = repo .ingest_pack(cm::cursor(corrupt), opts(false, None)) .await - .err() - .expect("corrupt"); + .expect_err("corrupt"); let s = err.to_string(); assert!( s.contains("index-pack") @@ -452,8 +469,7 @@ async fn ingest_failures_name_the_cause_and_leave_nothing_behind() { let err = repo .ingest_pack(cm::cursor(thin.clone()), opts(true, None)) .await - .err() - .expect("no base"); + .expect_err("no base"); assert!(err.to_string().contains("index-pack"), "{err}"); assert_eq!(pack_count(), 0); @@ -461,8 +477,7 @@ async fn ingest_failures_name_the_cause_and_leave_nothing_behind() { let err = repo .ingest_pack(cm::cursor(thin), opts(false, None)) .await - .err() - .expect("thin without fix-thin"); + .expect_err("thin without fix-thin"); assert!(err.to_string().contains("index-pack"), "{err}"); assert_eq!(pack_count(), 0); @@ -472,8 +487,7 @@ async fn ingest_failures_name_the_cause_and_leave_nothing_behind() { let err = repo .ingest_pack(cm::cursor(pack.clone()), opts(false, None)) .await - .err() - .expect("fsck"); + .expect_err("fsck"); assert!(err.to_string().contains("index-pack"), "{err}"); assert_eq!(pack_count(), 0); // …and accepted with fsck off (the knob is `wal.fsck_objects`). diff --git a/crates/walgit-git/tests/ls_refs.rs b/crates/walgit-git/tests/ls_refs.rs index f6a350b..4f09d90 100644 --- a/crates/walgit-git/tests/ls_refs.rs +++ b/crates/walgit-git/tests/ls_refs.rs @@ -1,3 +1,6 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::unwrap_used)] + mod common; use walgit_git::{LocalRepo, LsRefsArgs, ObjectFormat, RepoId, gix_hash}; diff --git a/crates/walgit-git/tests/refs.rs b/crates/walgit-git/tests/refs.rs index 7ff69f7..cf1f2f8 100644 --- a/crates/walgit-git/tests/refs.rs +++ b/crates/walgit-git/tests/refs.rs @@ -255,7 +255,7 @@ fn ref_view_lookups_are_logarithmic_and_overlay_aware() { let mut view = RefView::new(snap.clone()); assert_eq!( view.get("refs/heads/ref-123456").as_deref(), - Some(format!("{:040x}", 123456).as_str()) + Some(format!("{:040x}", 123_456).as_str()) ); assert_eq!( view.get("HEAD").as_deref(), diff --git a/crates/walgit-git/tests/refs500k.rs b/crates/walgit-git/tests/refs500k.rs index d4239b0..c946326 100644 --- a/crates/walgit-git/tests/refs500k.rs +++ b/crates/walgit-git/tests/refs500k.rs @@ -1,3 +1,10 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow( + clippy::unwrap_used, + clippy::ignore_without_reason, + clippy::used_underscore_binding +)] + //! `cargo test -p walgit-git --test refs500k -- --ignored --nocapture`: the per-push ref //! bookkeeping at 500 k refs (AGENTS §1.4: cost must not scale with ref count on a hot path). use std::io::Write; @@ -87,9 +94,13 @@ fn fixture(n_heads: usize, n_tags: usize) -> (tempfile::TempDir, LocalRepo) { names.sort(); for n in &names { if n.starts_with("refs/tags/") { - packed.push_str(&format!("{tag} {n}\n^{c}\n")); + { + let _ = std::fmt::Write::write_fmt(&mut packed, format_args!("{tag} {n}\n^{c}\n")); + }; } else { - packed.push_str(&format!("{c} {n}\n")); + { + let _ = std::fmt::Write::write_fmt(&mut packed, format_args!("{c} {n}\n")); + }; } } std::fs::write(dir.join("packed-refs"), packed).unwrap(); @@ -111,7 +122,7 @@ fn txn(name: &str, old: &str, new: &str) -> walgit_proto::v1::RefTransaction { } #[test] -#[ignore] +#[ignore = "500k ref benchmark; run in test-slow tier"] fn push_bookkeeping_at_500k_refs() { let (_root, repo) = fixture(400_000, 100_000); let c2 = commit(repo.path(), "two"); diff --git a/crates/walgit-git/tests/rev_index.rs b/crates/walgit-git/tests/rev_index.rs index ea7adce..fa9ce4b 100644 --- a/crates/walgit-git/tests/rev_index.rs +++ b/crates/walgit-git/tests/rev_index.rs @@ -1,3 +1,6 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::expect_used)] + //! `.rev` derived from the `.idx` alone must be byte-identical to git's //! (`index-pack --rev-index`), so a pack can get its reverse index in seconds //! (a large repository's 32 GB base: `index-pack --rev-index` re-reads the whole pack — diff --git a/crates/walgit-git/tests/upload_gix_remote.rs b/crates/walgit-git/tests/upload_gix_remote.rs index 6999aa6..e65e0fc 100644 --- a/crates/walgit-git/tests/upload_gix_remote.rs +++ b/crates/walgit-git/tests/upload_gix_remote.rs @@ -1,3 +1,6 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::unwrap_used)] + //! The gix upload-pack engine over a repository whose base pack is *not* //! local: history from the commit-graph chain, `have`s from the faulter's //! index, object enumeration by tree diff against parents, base objects diff --git a/crates/walgit-git/tests/upload_gix_scale.rs b/crates/walgit-git/tests/upload_gix_scale.rs index 6d77f23..e1b9f10 100644 --- a/crates/walgit-git/tests/upload_gix_scale.rs +++ b/crates/walgit-git/tests/upload_gix_scale.rs @@ -1,3 +1,12 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow( + clippy::expect_used, + clippy::indexing_slicing, + clippy::unwrap_used, + clippy::cast_possible_truncation, + clippy::format_push_string +)] + //! Reproducer for AGENTS §6 "gix large-fetch object-id corruption and 178 GB OOM" (2026-08-21 //! 05:4xZ: a remainder pack carried an entry under another object's id; 07:0xZ: the same shape //! replayed over a large repository was OOM-killed at 178 GB anon RSS after `Enumerating objects: 113683`). @@ -8,7 +17,7 @@ //! A. remainder: want tip, have base tip, `thin_pack = true` (prod's failing shape), //! B. remainder, `thin_pack = false` (every delta whose base is outside the set re-encoded), //! C. bounded zero-have: `--depth=1 --filter=blob:none` (CI's shape), -//! D. full zero-have (TreeContents expansion). +//! D. full zero-have (`TreeContents` expansion). //! Every output is indexed by stock git with `--strict` (ids recomputed from content), its object //! set compared to `git rev-list --objects` of the source, and the process's max RSS delta is //! bounded by a small multiple of the pack bytes. @@ -29,15 +38,25 @@ mod cm { } fn max_rss_kb() -> u64 { + // SAFETY: rusage contains C numeric fields whose all-zero values are valid. + #[allow(unsafe_code)] let mut ru: libc::rusage = unsafe { std::mem::zeroed() }; - unsafe { libc::getrusage(libc::RUSAGE_SELF, &mut ru) }; + // SAFETY: ru is aligned writable storage; RUSAGE_SELF is a supported selector. + #[allow(unsafe_code)] + let result = unsafe { libc::getrusage(libc::RUSAGE_SELF, &raw mut ru) }; + assert_eq!( + result, + 0, + "getrusage failed: {}", + std::io::Error::last_os_error() + ); // getrusage reports ru_maxrss in KB on Linux but in BYTES on macOS/BSD. // Without this, the memory-bound assertion reads 1024x high on macOS and // fails a passing result (a 16 MB delta shown as "16832 MB"). #[cfg(any(target_os = "macos", target_os = "ios"))] - let kb = (ru.ru_maxrss as u64) / 1024; + let kb = (u64::try_from(ru.ru_maxrss).expect("nonnegative peak RSS")) / 1024; #[cfg(not(any(target_os = "macos", target_os = "ios")))] - let kb = ru.ru_maxrss as u64; + let kb = u64::try_from(ru.ru_maxrss).expect("nonnegative peak RSS"); kb } @@ -59,7 +78,7 @@ fn synth(commits: usize, files: usize, files_per_commit: usize, dirs: usize) -> { let stdin = child.stdin.as_mut().unwrap(); let mut w = std::io::BufWriter::with_capacity(1 << 20, stdin); - let mut seed = 0x9E3779B97F4A7C15u64; + let mut seed = 0x9E37_79B9_7F4A_7C15_u64; let mut next = || { seed ^= seed << 13; seed ^= seed >> 7; @@ -73,7 +92,12 @@ fn synth(commits: usize, files: usize, files_per_commit: usize, dirs: usize) -> let mut c = format!("file {f}\n"); let words = 200 + (next() % 6000) as usize; for _ in 0..words { - c.push_str(&format!("{:06x} ", next() & 0xffffff)); + { + let _ = std::fmt::Write::write_fmt( + &mut c, + format_args!("{:06x} ", next() & 0x00ff_ffff), + ); + }; } c.push('\n'); c @@ -214,7 +238,7 @@ fn req( } } -/// Entries of type REF_DELTA (7) in a v2 pack: walk the headers, skipping compressed data with a +/// Entries of type `REF_DELTA` (7) in a v2 pack: walk the headers, skipping compressed data with a /// throwaway inflater. A self-contained pack written by pack-copy has none. fn count_ref_deltas(pack: &[u8]) -> usize { use std::io::Read; @@ -508,7 +532,7 @@ async fn gix_engine_packs_are_strict_valid_and_bounded_in_memory_30k() { /// ~300 k objects with long delta chains across two packs: `just test-slow`. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[ignore] +#[ignore = "large stress test; run in test-slow tier"] async fn gix_engine_packs_are_strict_valid_and_bounded_in_memory_300k() { run_shapes(12_000, 1_500, 10, 40, 10_000).await; } diff --git a/crates/walgit-git/tests/upload_pack.rs b/crates/walgit-git/tests/upload_pack.rs index af89da3..ab875d8 100644 --- a/crates/walgit-git/tests/upload_pack.rs +++ b/crates/walgit-git/tests/upload_pack.rs @@ -1,3 +1,6 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::unwrap_used, clippy::case_sensitive_file_extension_comparisons)] + mod common; use walgit_git::pkt::Protocol; @@ -672,10 +675,10 @@ async fn fetch_skips_gitlink_entries() { // present. let (_blobs, commits, trees, _tags) = cm::pack_object_types(&pack); assert_eq!(commits, 2, "unexpected commit count ({filter:?})"); - if filter != Some("tree:0") { - assert!(trees >= 1, "tree missing ({filter:?})"); - } else { + if filter == Some("tree:0") { assert_eq!(trees, 0, "tree:0 sends no trees"); + } else { + assert!(trees >= 1, "tree missing ({filter:?})"); } let tmp = cm::fresh_bare(); let pack_path = tmp.path().join("objects/pack/pack-test.pack"); @@ -699,7 +702,7 @@ async fn fetch_skips_gitlink_entries() { /// diff-sized fetch (want HEAD, have HEAD~50) and a full clone, both engines. /// `cargo test -p walgit-git --test upload_pack bench_fetch_engines -- --ignored --nocapture` #[tokio::test] -#[ignore] +#[ignore = "benchmark requires WALGIT_BENCH_REPO"] async fn bench_fetch_engines() { let Ok(src_path) = std::env::var("WALGIT_BENCH_REPO") else { eprintln!("WALGIT_BENCH_REPO not set; skipping"); diff --git a/crates/walgit-proto/proto/walgit/v1/wal.proto b/crates/walgit-proto/proto/walgit/v1/wal.proto index f9e3c9f..9da1199 100644 --- a/crates/walgit-proto/proto/walgit/v1/wal.proto +++ b/crates/walgit-proto/proto/walgit/v1/wal.proto @@ -41,7 +41,7 @@ message Manifest { // Log segments covering [min_seq, head_seq], ascending, contiguous, // non-overlapping. Kept short by merging segments during compaction. repeated LogSegmentRef log_segments = 7; - // Denormalized live pack set after applying every entry <= head_seq + // Denormalized live pack set after applying every entry <= `head_seq` // (checkpoint packs + packs of later PUSH entries − superseded). Sorted by // seq. Materialize = these packs + checkpoint refs + replay log > checkpoint. repeated PackRef packs = 8; @@ -86,7 +86,7 @@ message LogSegmentRef { } // Body of a log object. Encoding on the wire is a sequence of length-prefixed -// frames (uvarint len + LogEntry bytes) so appendable objects can grow without +// frames (uvarint len + `LogEntry` bytes) so appendable objects can grow without // rewriting; `LogSegment` is the in-memory/decoded form and the encoding used // for sealed immutable segments written whole. message LogSegment { @@ -143,7 +143,7 @@ message LogEntry { EntryKind kind = 2; // Present for PUSH (when objects were pushed) and COMPACT. PackRef pack = 3; - // Present for PUSH and REF_UPDATE. + // Present for PUSH and `REF_UPDATE`. RefTransaction txn = 4; // COMPACT only: pack checksums removed from the live set by this entry. repeated string supersedes = 5; @@ -153,7 +153,7 @@ message LogEntry { string writer = 8; // Free-form provenance (push-options, client agent, principal). Small. map meta = 9; - // ENTRY_KIND_SETTINGS: the settings as published at this seq. + // `ENTRY_KIND_SETTINGS`: the settings as published at this seq. RepoSettings settings = 10; } @@ -184,7 +184,7 @@ message Checkpoint { // Packs that fully represent the repository at `seq` (typically 1 base pack // (+ 1 medium)). Keys are wal/.pack. repeated PackRef packs = 3; - // Key of the RefSnapshot, e.g. "checkpoints//refs.pb". + // Key of the `RefSnapshot`, e.g. "checkpoints//refs.pb". string refs_key = 4; uint64 ref_count = 5; // Optional rendered full bundle for bundle-uri, e.g. "checkpoints//.bundle". @@ -201,12 +201,12 @@ message CheckpointRef { // fetching the checkpoint object). google.protobuf.Timestamp created_at = 3; // Earliest WAL state this repository ever had (carried forward from the - // previous checkpoint, else the first folded entry's created_at): bundle + // previous checkpoint, else the first folded entry's `created_at`: bundle // slots before it are "unavailable"; slots after it are backfillable even // on a maintainer that cold-starts from this checkpoint (D22). google.protobuf.Timestamp first_state_at = 4; - // created_at of the newest entry this checkpoint folded: the state it holds - // is the repository "as of" this instant (≠ created_at, the write time). + // `created_at` of the newest entry this checkpoint folded: the state it holds + // is the repository "as of" this instant (≠ `created_at`, the write time). google.protobuf.Timestamp as_of = 5; } @@ -252,7 +252,7 @@ message BundleList { repeated BundleEntry bundles = 3; google.protobuf.Timestamp updated_at = 4; // Closed slots measured and NOT cut (too small / no state as of the slot): - // final for (strategy, slot, base_id) — every host and every restart skips + // final for (strategy, slot, `base_id`) — every host and every restart skips // them in O(1) instead of re-measuring (a unit's worth of work each; after // a restart the SSD host re-walked ~30 of them before reaching the live slot, // 2026-08-21). A new base bundle for the slot re-opens the question. @@ -287,7 +287,7 @@ message BundleEntry { // Id of the bundle this incremental one is based on (empty for full). string base_id = 8; google.protobuf.Timestamp created_at = 9; - // Object store version tag (ETag/generation) at upload; used for HTTP ETag. + // Object store version tag (ETag/generation) at upload; used for HTTP `ETag`. string version = 10; // Ref tips the bundle contains (refs/heads/*, refs/tags/*, HEAD). For // incremental bundles, the base bundle's tips are the prerequisites. diff --git a/crates/walgit-proto/src/lib.rs b/crates/walgit-proto/src/lib.rs index 128f6f5..55c696d 100644 --- a/crates/walgit-proto/src/lib.rs +++ b/crates/walgit-proto/src/lib.rs @@ -3,6 +3,8 @@ //! Schema lives in `proto/walgit/v1/wal.proto`; it is the contract between //! every walgit instance and must only evolve backward-compatibly. +// Documentation in this module is emitted by prost, including enum helper prose. +#[allow(clippy::doc_markdown)] pub mod v1 { include!(concat!(env!("OUT_DIR"), "/walgit.v1.rs")); } @@ -95,7 +97,7 @@ pub mod keys { /// Appendable objects grow by appending frames; readers stop at the first /// incomplete trailing frame. pub mod frame { - use bytes::{Buf, Bytes, BytesMut}; + use bytes::{Bytes, BytesMut}; use prost::Message; use crate::v1::LogEntry; @@ -104,7 +106,8 @@ pub mod frame { let len = e.encoded_len(); prost::encoding::encode_varint(len as u64, out); out.reserve(len); - e.encode(out).expect("BytesMut has capacity"); + // BytesMut grows as needed; encode_raw has no fallible capacity check. + e.encode_raw(out); } pub fn encode_entries<'a>(entries: impl IntoIterator) -> Bytes { @@ -120,16 +123,17 @@ pub mod frame { pub fn decode_entries(buf: &[u8]) -> Result<(Vec, usize), prost::DecodeError> { let mut out = Vec::new(); let mut pos = 0usize; - loop { - let mut probe = &buf[pos..]; + while let Some(mut probe) = buf.get(pos..) { let Ok(len) = prost::encoding::decode_varint(&mut probe) else { break; }; - let len = len as usize; - if probe.remaining() < len { + let Ok(len) = usize::try_from(len) else { break; - } - out.push(LogEntry::decode(&probe[..len])?); + }; + let Some(frame) = probe.get(..len) else { + break; + }; + out.push(LogEntry::decode(frame)?); pos = buf.len() - probe.len() + len; } Ok((out, pos)) @@ -146,12 +150,16 @@ pub mod time { pub fn from_system(t: SystemTime) -> prost_types::Timestamp { let d = t.duration_since(UNIX_EPOCH).unwrap_or_default(); prost_types::Timestamp { - seconds: d.as_secs() as i64, - nanos: d.subsec_nanos() as i32, + seconds: i64::try_from(d.as_secs()).unwrap_or(i64::MAX), + nanos: i32::try_from(d.subsec_nanos()).unwrap_or(999_999_999), } } pub fn to_system(t: &prost_types::Timestamp) -> SystemTime { - UNIX_EPOCH + Duration::new(t.seconds.max(0) as u64, t.nanos.max(0) as u32) + UNIX_EPOCH + + Duration::new( + t.seconds.max(0).cast_unsigned(), + t.nanos.max(0).cast_unsigned(), + ) } } diff --git a/crates/walgit-server/build.rs b/crates/walgit-server/build.rs index f7a7eb4..2df6dee 100644 --- a/crates/walgit-server/build.rs +++ b/crates/walgit-server/build.rs @@ -10,19 +10,20 @@ use std::path::Path; const PLACEHOLDER: &str = "\nwalgit\n\

walgit web UI is not built in this binary. Run just web-build (vite via pnpm) and rebuild.

\n"; -fn main() { +fn main() -> std::io::Result<()> { println!("cargo:rustc-env=WALGIT_BUILD_SHA={}", build_sha()); let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); let dist = manifest.join("../../web/dist"); println!("cargo:rerun-if-changed={}", dist.display()); let index = dist.join("index.html"); if !index.exists() { - fs::create_dir_all(&dist).expect("create web/dist"); - fs::write(&index, PLACEHOLDER).expect("write placeholder web/dist/index.html"); + fs::create_dir_all(&dist)?; + fs::write(&index, PLACEHOLDER)?; println!( "cargo:warning=web/dist was missing; wrote a placeholder index.html (run `just web-build` for the real UI)" ); } + Ok(()) } /// Build identity for `/healthz` (`version`) and `walgit --version`: the commit diff --git a/crates/walgit-server/src/auth.rs b/crates/walgit-server/src/auth.rs index 78d4312..218ea69 100644 --- a/crates/walgit-server/src/auth.rs +++ b/crates/walgit-server/src/auth.rs @@ -3,7 +3,7 @@ //! //! * **`token`** — static tokens from the config, presented as `Authorization: //! Bearer ` or as the password of HTTP Basic (any user name). -//! * **`oidc`** — any OpenID Connect issuer. Three credentials are accepted: +//! * **`oidc`** — any `OpenID` Connect issuer. Three credentials are accepted: //! 1. an **ID token** from the issuer in `Authorization: Bearer` (RS256/ES256, //! signature against the issuer's JWKS, `iss`, `exp`, `aud` ∈ `audiences` ∪ //! {`oauth_client_id`}, `email_verified`), for CLIs that can mint one; @@ -216,7 +216,7 @@ impl JwksSource for HttpOidcSource { .get(reqwest::header::CACHE_CONTROL) .and_then(|v| v.to_str().ok()) .and_then(parse_max_age) - .unwrap_or(Duration::from_secs(300)); + .unwrap_or(Duration::from_mins(5)); let document: JwksDocument = response .error_for_status() .map_err(|e| format!("JWKS response failed: {e}"))? @@ -921,7 +921,7 @@ fn bearer_token(headers: &HeaderMap) -> Option { /// Value of cookie `name` from the `Cookie` header(s). pub fn cookie_value(headers: &HeaderMap, name: &str) -> Option { - for h in headers.get_all(axum::http::header::COOKIE).iter() { + for h in &headers.get_all(axum::http::header::COOKIE) { let Ok(s) = h.to_str() else { continue }; for part in s.split(';') { let part = part.trim(); @@ -1049,7 +1049,7 @@ mod tests { } // gitleaks:allow — fixed test fixture; never loaded outside this module's OIDC verifier tests. - const PRIVATE_KEY: &[u8] = br#"-----BEGIN PRIVATE KEY----- + const PRIVATE_KEY: &[u8] = br"-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDJETqse41HRBsc 7cfcq3ak4oZWFCoZlcic525A3FfO4qW9BMtRO/iXiyCCHn8JhiL9y8j5JdVP2Q9Z IpfElcFd3/guS9w+5RqQGgCR+H56IVUyHZWtTJbKPcwWXQdNUX0rBFcsBzCRESJL @@ -1077,7 +1077,7 @@ GcZ0izY/30012ajdHY+/QK5lsMoxTnn0skdS+spLxaS5ZEO4qvPVb8RAoCkWMMal 2pOhmquJQVDPDLuZHdrIiKiDM20dy9sMfHygWcZjQ4WSxf/J7T9canLZIXFhHAZT 3wc9h4G8BBCtWN2TN/LsGZdB -----END PRIVATE KEY----- -"#; +"; const MODULUS: &str = "yRE6rHuNR0QbHO3H3Kt2pOKGVhQqGZXInOduQNxXzuKlvQTLUTv4l4sggh5_CYYi_cvI-SXVT9kPWSKXxJXBXd_4LkvcPuUakBoAkfh-eiFVMh2VrUyWyj3MFl0HTVF9KwRXLAcwkREiS3npThHRyIxuy0ZMeZfxVL5arMhw1SRELB8HoGfG_AtH89BIE9jDBHZ9dLelK9a184zAf8LwoPLxvJb3Il5nncqPcSfKDDodMFBIMc4lQzDKL5gvmiXLXB1AGLm8KBjfE8s3L5xqi-yUod-j8MtvIj812dkS4QMiRVN_by2h3ZY8LYVGrqZXZTcgn2ujn8uKjXLZVD5TdQ"; const EXPONENT: &str = "AQAB"; @@ -1131,7 +1131,7 @@ GcZ0izY/30012ajdHY+/QK5lsMoxTnn0skdS+spLxaS5ZEO4qvPVb8RAoCkWMMal n: MODULUS.into(), e: EXPONENT.into(), }], - max_age: Duration::from_secs(3600), + max_age: Duration::from_hours(1), })), }) } @@ -1322,7 +1322,7 @@ GcZ0izY/30012ajdHY+/QK5lsMoxTnn0skdS+spLxaS5ZEO4qvPVb8RAoCkWMMal async fn issued_access_tokens_are_bearers_and_basic_passwords_and_never_cookies() { let mut cfg = config(); cfg.server.auth.session_secret = Some(SECRET.into()); - cfg.server.auth.access_token_ttl = Duration::from_secs(3600); + cfg.server.auth.access_token_ttl = Duration::from_hours(1); let auth = Authenticator::with_key_source(&cfg, source()); let tok = auth.access_token("dev@example.com").unwrap(); assert!(tok.starts_with(ACCESS_TOKEN_PREFIX)); @@ -1431,7 +1431,7 @@ mod session_tests { assert!(unix_now().unwrap().abs_diff(iat) <= 2); assert_eq!( walgit_config::Config::default().server.auth.session_ttl, - Duration::from_secs(30 * 86400) + Duration::from_hours(720) ); } diff --git a/crates/walgit-server/src/bridge.rs b/crates/walgit-server/src/bridge.rs index 30e32cc..349ff18 100644 --- a/crates/walgit-server/src/bridge.rs +++ b/crates/walgit-server/src/bridge.rs @@ -161,7 +161,7 @@ impl Bridge { // Another bridge instance advanced it: our emission was a // duplicate (dedup key), theirs stands. Err(StoreError::PreconditionFailed { .. }) => { - tracing::warn!(repo = %id, "events bridge: cursor CAS lost (two bridges?)") + tracing::warn!(repo = %id, "events bridge: cursor CAS lost (two bridges?)"); } Err(e) => return Err(e.into()), } @@ -195,7 +195,7 @@ impl Bridge { } Ok(_) => {} Err(e) => { - tracing::warn!(repo = %id, error = %e, "events bridge: sweep catch-up failed") + tracing::warn!(repo = %id, error = %e, "events bridge: sweep catch-up failed"); } } } diff --git a/crates/walgit-server/src/bundles.rs b/crates/walgit-server/src/bundles.rs index 7b23cc7..d9fc0f1 100644 --- a/crates/walgit-server/src/bundles.rs +++ b/crates/walgit-server/src/bundles.rs @@ -1,6 +1,6 @@ //! Bundle serving: `GET /{repo}/bundles/list` (git bundle-list text, no-cache) //! and `GET|HEAD /{repo}/bundles/{strategy}/{name}` (streamed bundle with -//! strong ETag = store version, immutable caching, Range/If-Range, +//! strong `ETag` = store version, immutable caching, Range/If-Range, //! If-None-Match, HEAD — `static_object`). use axum::http::{HeaderMap, Method, StatusCode}; @@ -106,7 +106,7 @@ fn render_bundle_list_response(text: String) -> Response { } /// `GET|HEAD /{repo}/bundles/{strategy}/{name}` — streamed from the store -/// with the full immutable-object contract (strong ETag, 304, Range/If-Range, +/// with the full immutable-object contract (strong `ETag`, 304, Range/If-Range, /// HEAD, Content-Length); see `static_object`. pub async fn object( st: &AppState, @@ -196,18 +196,17 @@ pub async fn compose_full_from_base( // rig's weekly compose failed every pass for as long as the churn kept refs moving, 2026-08-22); // only a log folded away below the base's seq with no checkpoint before it is unrecoverable. let refs_key = walgit_proto::keys::checkpoint_refs_key(seq); - let snap = match store.get_bytes(&refs_key).await? { - Some((_, bytes)) => walgit_proto::v1::RefSnapshot::decode(bytes.as_ref())?, - None => { - info!( - base_seq = seq, - head = manifest.head_seq, - "no checkpoint at the base's seq: replaying the refs at that seq from the WAL for the compose" - ); - handle.refs_at_seq(seq).await.map_err(|e| { - anyhow::anyhow!("refs at the base's seq {seq} (head {}): {e} — run `walgit compact --base` again so a checkpoint exists at the base", manifest.head_seq) - })? - } + let snap = if let Some((_, bytes)) = store.get_bytes(&refs_key).await? { + walgit_proto::v1::RefSnapshot::decode(bytes.as_ref())? + } else { + info!( + base_seq = seq, + head = manifest.head_seq, + "no checkpoint at the base's seq: replaying the refs at that seq from the WAL for the compose" + ); + handle.refs_at_seq(seq).await.map_err(|e| { + anyhow::anyhow!("refs at the base's seq {seq} (head {}): {e} — run `walgit compact --base` again so a checkpoint exists at the base", manifest.head_seq) + })? }; let list = walgit_bundle::ops::read_list(store) .await? diff --git a/crates/walgit-server/src/cache.rs b/crates/walgit-server/src/cache.rs index 4327ebf..06c08bf 100644 --- a/crates/walgit-server/src/cache.rs +++ b/crates/walgit-server/src/cache.rs @@ -4,11 +4,11 @@ //! Each cache exposes hit/miss counters via the `metrics` crate. //! //! **Justification for `moka`:** these caches need bounded, concurrent, -//! size-based LRU eviction. Implementing LRU eviction on DashMap requires a +//! size-based LRU eviction. Implementing LRU eviction on `DashMap` requires a //! secondary ordering structure and manual locking — error-prone and slower. //! `moka::sync::Cache` provides thread-safe, size-bounded LRU out of the box //! with excellent throughput (bucket-level locking, no global lock on hot -//! path). DashMap remains the right choice for unbounded lookup tables +//! path). `DashMap` remains the right choice for unbounded lookup tables //! (e.g. `RepoSemaphores`); bounded LRU is moka's domain. use moka::sync::Cache; @@ -61,7 +61,7 @@ fn v2_key(repo: &str, version: Option<&Version>, args: &walgit_git::LsRefsArgs) } /// Cache for rendered v0 ref advertisements. -/// Keyed by (repo, manifest_version, service). +/// Keyed by (repo, `manifest_version`, service). #[derive(Clone)] pub struct RefAdvertCache { v0: Cache>, @@ -83,15 +83,12 @@ impl RefAdvertCache { service: walgit_git::Service, ) -> Option> { let key = v0_key(repo, version, service); - match self.v0.get(&key) { - Some(val) => { - metrics::counter!("walgit_cache_ref_advert_hit").increment(1); - Some(val) - } - None => { - metrics::counter!("walgit_cache_ref_advert_miss").increment(1); - None - } + if let Some(val) = self.v0.get(&key) { + metrics::counter!("walgit_cache_ref_advert_hit").increment(1); + Some(val) + } else { + metrics::counter!("walgit_cache_ref_advert_miss").increment(1); + None } } @@ -114,15 +111,12 @@ impl RefAdvertCache { args: &walgit_git::LsRefsArgs, ) -> Option> { let key = v2_key(repo, version, args); - match self.v2_ls_refs.get(&key) { - Some(val) => { - metrics::counter!("walgit_cache_ls_refs_hit").increment(1); - Some(val) - } - None => { - metrics::counter!("walgit_cache_ls_refs_miss").increment(1); - None - } + if let Some(val) = self.v2_ls_refs.get(&key) { + metrics::counter!("walgit_cache_ls_refs_hit").increment(1); + Some(val) + } else { + metrics::counter!("walgit_cache_ls_refs_miss").increment(1); + None } } @@ -163,7 +157,7 @@ pub struct BundleListCache { } /// Idle lifetime of a rendered list (freshness comes from the version key). -pub const BUNDLE_LIST_TTL: std::time::Duration = std::time::Duration::from_secs(600); +pub const BUNDLE_LIST_TTL: std::time::Duration = std::time::Duration::from_mins(10); impl BundleListCache { pub fn new(max_entries: usize) -> Self { @@ -185,15 +179,12 @@ impl BundleListCache { repo: repo.to_string(), list_version: list_version.to_string(), }; - match self.inner.get(&key) { - Some(val) => { - metrics::counter!("walgit_cache_bundle_list_hit").increment(1); - Some(val) - } - None => { - metrics::counter!("walgit_cache_bundle_list_miss").increment(1); - None - } + if let Some(val) = self.inner.get(&key) { + metrics::counter!("walgit_cache_bundle_list_hit").increment(1); + Some(val) + } else { + metrics::counter!("walgit_cache_bundle_list_miss").increment(1); + None } } @@ -298,7 +289,7 @@ struct RefIndexKey { version: String, } -/// Keyed by (repo, manifest_version). +/// Keyed by (repo, `manifest_version`). #[derive(Clone)] pub struct RefIndexCache { inner: Cache>, @@ -366,7 +357,7 @@ impl ServerCaches { .build(), bundle_attempts: Cache::builder() .max_capacity(100_000) - .time_to_live(std::time::Duration::from_secs(6 * 3600)) + .time_to_live(std::time::Duration::from_hours(6)) .build(), } } @@ -383,7 +374,10 @@ mod tests { fn make_args(prefixes: &[&str]) -> walgit_git::LsRefsArgs { walgit_git::LsRefsArgs { - ref_prefixes: prefixes.iter().map(|s| s.to_string()).collect(), + ref_prefixes: prefixes + .iter() + .map(std::string::ToString::to_string) + .collect(), symrefs: false, peel: true, unborn: false, @@ -552,7 +546,7 @@ mod tests { } /// Benchmark: measure ref advertisement render time with and without cache - /// for a 50k-ref repo. Run with: cargo test -p walgit-server bench_ref_advert -- --nocapture --ignored + /// for a 50k-ref repo. Run with: cargo test -p walgit-server `bench_ref_advert` -- --nocapture --ignored #[test] #[ignore = "requires git binary and takes ~10s"] fn bench_ref_advert_50k_refs() { diff --git a/crates/walgit-server/src/events.rs b/crates/walgit-server/src/events.rs index e11cc99..4d4d77b 100644 --- a/crates/walgit-server/src/events.rs +++ b/crates/walgit-server/src/events.rs @@ -1,7 +1,7 @@ //! `ref` event shapes and the WAL → event conversion. Contract: //! `docs/EVENTS.md`. The only producer is the bridge (`crate::bridge`): it //! tails each repo's WAL from a durable cursor, converts committed PUSH / -//! REF_UPDATE entries with [`refs_from_entries`], and delivers to every +//! `REF_UPDATE` entries with [`refs_from_entries`], and delivers to every //! [`Sink`] before advancing the cursor. Nothing on the push path knows events //! exist. //! @@ -111,7 +111,7 @@ impl RefEvent { } } -/// `ref` events for the PUSH / REF_UPDATE entries in `entries`, in seq order. +/// `ref` events for the PUSH / `REF_UPDATE` entries in `entries`, in seq order. pub(crate) fn refs_from_entries(repo: &RepoId, entries: &[LogEntry], out: &mut Vec) { let repo = repo.to_string(); for entry in entries { @@ -177,7 +177,7 @@ impl WebhookSink { pub fn new(url: String, secret: Option) -> Self { WebhookSink { url, - secret: secret.map(|s| s.into_bytes()), + secret: secret.map(std::string::String::into_bytes), client: reqwest::Client::builder() .timeout(std::time::Duration::from_secs(10)) .build() diff --git a/crates/walgit-server/src/follow.rs b/crates/walgit-server/src/follow.rs index 5a97991..261399a 100644 --- a/crates/walgit-server/src/follow.rs +++ b/crates/walgit-server/src/follow.rs @@ -170,7 +170,7 @@ pub async fn run_pass(state: &Arc) -> anyhow::Result { let repo = id.to_string(); match fetched { Ok((false, tips, have)) => { - debug!(repo = %id, %upstream, elapsed_ms = t0.elapsed().as_millis() as u64, "follow: in sync"); + debug!(repo = %id, %upstream, elapsed_ms = u64::try_from(t0.elapsed().as_millis()).unwrap_or(u64::MAX), "follow: in sync"); state.follow.set( &repo, "in-sync", @@ -186,57 +186,60 @@ pub async fn run_pass(state: &Arc) -> anyhow::Result { report.behind += 1; let mut params = HashMap::new(); params.insert("prefetched".to_string(), "1".to_string()); - match run_op(state, &id, params).await { - Some(v) => { - let n = v.get("published").and_then(|n| n.as_u64()).unwrap_or(0); - if n > 0 { - report.published += 1; - } - let seq = v.get("seq").and_then(|s| s.as_u64()).unwrap_or(0); - let refused: Vec = v - .get("refused") - .and_then(|r| r.as_array()) - .map(|a| { - a.iter() - .filter_map(|x| x.as_str().map(String::from)) - .collect() - }) - .unwrap_or_default(); - let detail = if refused.is_empty() { - format!("{n} ref(s) published at seq {seq}") - } else { - format!( - "{n} ref(s) published at seq {seq}; refused: {}", - refused.join("; ") - ) - }; - state.follow.set( - &repo, - if n > 0 { "published" } else { "refused" }, - detail, - tips, - have, - ); - } - None => { - report.failed += 1; - // The task's summary names the reason (rewind, unpack, connectivity, publish). - let why = state - .registry - .tasks() - .recent(&repo) - .into_iter() - .find(|t| t.kind == "follow") - .map(|t| t.summary) - .unwrap_or_default(); - state.follow.set(&repo, "refused", why, tips, have); + if let Some(v) = run_op(state, &id, params).await { + let n = v + .get("published") + .and_then(serde_json::Value::as_u64) + .unwrap_or(0); + if n > 0 { + report.published += 1; } + let seq = v + .get("seq") + .and_then(serde_json::Value::as_u64) + .unwrap_or(0); + let refused: Vec = v + .get("refused") + .and_then(|r| r.as_array()) + .map(|a| { + a.iter() + .filter_map(|x| x.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + let detail = if refused.is_empty() { + format!("{n} ref(s) published at seq {seq}") + } else { + format!( + "{n} ref(s) published at seq {seq}; refused: {}", + refused.join("; ") + ) + }; + state.follow.set( + &repo, + if n > 0 { "published" } else { "refused" }, + detail, + tips, + have, + ); + } else { + report.failed += 1; + // The task's summary names the reason (rewind, unpack, connectivity, publish). + let why = state + .registry + .tasks() + .recent(&repo) + .into_iter() + .find(|t| t.kind == "follow") + .map(|t| t.summary) + .unwrap_or_default(); + state.follow.set(&repo, "refused", why, tips, have); } } Err(e) => { report.failed += 1; metrics::counter!("walgit_follow_rounds_total", "repo" => id.to_string(), "outcome" => "fetch-failed").increment(1); - warn!(repo = %id, %upstream, error = format!("{e:#}"), elapsed_ms = t0.elapsed().as_millis() as u64, "follow: fetch from upstream failed"); + warn!(repo = %id, %upstream, error = format!("{e:#}"), elapsed_ms = u64::try_from(t0.elapsed().as_millis()).unwrap_or(u64::MAX), "follow: fetch from upstream failed"); state.follow.set( &repo, "failed", @@ -257,11 +260,10 @@ async fn run_op( params: HashMap, ) -> Option { let task = match crate::ops::start(state.clone(), id.clone(), "follow", params).await { - Ok(t) => t, - Err(crate::ops::StartError::AlreadyRunning(t)) => t, + Ok(t) | Err(crate::ops::StartError::AlreadyRunning(t)) => t, Err(crate::ops::StartError::UnknownOp) => return None, }; - if !task.wait_done(std::time::Duration::from_secs(3600)).await { + if !task.wait_done(std::time::Duration::from_hours(1)).await { warn!(repo = %id, "follow: op still running after 1h; moving on"); return None; } @@ -344,7 +346,7 @@ pub(crate) async fn op( // completed it from our own objects, so it is not thin). let ingested = match &delta.pack { Some(p) => { - let bytes = tokio::fs::metadata(p).await.map(|m| m.len()).unwrap_or(0); + let bytes = tokio::fs::metadata(p).await.map_or(0, |m| m.len()); log(format!("ingesting {bytes} bytes of objects from upstream")); let file = tokio::fs::File::open(p) .await @@ -435,8 +437,7 @@ pub(crate) async fn op( let (old, new) = planned .iter() .find(|(n, _, _)| n == name) - .map(|(_, o, n)| (o.as_str(), n.as_str())) - .unwrap_or(("", "")); + .map_or(("", ""), |(_, o, n)| (o.as_str(), n.as_str())); match r { Ok(()) => { published += 1; @@ -452,7 +453,7 @@ pub(crate) async fn op( } metrics::counter!("walgit_follow_rounds_total", "repo" => id.to_string(), "outcome" => "published").increment(1); metrics::counter!("walgit_follow_refs_total", "repo" => id.to_string()).increment(published); - info!(repo = %id, seq = res.seq, refs = published, refused = refused.len(), %upstream, elapsed_ms = t0.elapsed().as_millis() as u64, "follow published"); + info!(repo = %id, seq = res.seq, refs = published, refused = refused.len(), %upstream, elapsed_ms = u64::try_from(t0.elapsed().as_millis()).unwrap_or(u64::MAX), "follow published"); let summary = format!( "{published} ref(s) from upstream published at seq {} in {:.1}s{}", res.seq, diff --git a/crates/walgit-server/src/forward.rs b/crates/walgit-server/src/forward.rs index b3897c1..1b30ce6 100644 --- a/crates/walgit-server/src/forward.rs +++ b/crates/walgit-server/src/forward.rs @@ -52,9 +52,9 @@ pub async fn receive_pack( route.id.name() ); let client = reqwest::Client::new(); - let stream = body.into_data_stream().map(|chunk| { - chunk.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string())) - }); + let stream = body + .into_data_stream() + .map(|chunk| chunk.map_err(|e| std::io::Error::other(e.to_string()))); let mut request = client .post(&endpoint) .body(reqwest::Body::wrap_stream(stream)); @@ -81,22 +81,21 @@ pub async fn receive_pack( .ok() .filter(|v| !v.is_empty()) .or_else(|| broker_token.map(str::to_string).filter(|v| !v.is_empty())); - match token { - Some(token) => request = request.bearer_auth(token), - None => { - tracing::warn!( - elapsed_ms = started.elapsed().as_millis() as u64, - "push broker token unset (wal.push_broker_token / WALGIT_BROKER_TOKEN); falling back" - ); - return ForwardOutcome::Fallback; - } + if let Some(token) = token { + request = request.bearer_auth(token); + } else { + tracing::warn!( + elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), + "push broker token unset (wal.push_broker_token / WALGIT_BROKER_TOKEN); falling back" + ); + return ForwardOutcome::Fallback; } } let response = match request.send().await { Ok(response) => response, Err(error) => { - tracing::warn!(%error, elapsed_ms = started.elapsed().as_millis() as u64, "push broker unavailable; falling back"); + tracing::warn!(%error, elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), "push broker unavailable; falling back"); return ForwardOutcome::Fallback; } }; @@ -106,7 +105,7 @@ pub async fn receive_pack( ) { tracing::warn!( status = response.status().as_u16(), - elapsed_ms = started.elapsed().as_millis() as u64, + elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), "push broker gateway failure; falling back" ); return ForwardOutcome::Fallback; @@ -114,9 +113,9 @@ pub async fn receive_pack( let status = response.status(); let response_headers = response.headers().clone(); - let stream = response.bytes_stream().map(|chunk| { - chunk.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string())) - }); + let stream = response + .bytes_stream() + .map(|chunk| chunk.map_err(|e| std::io::Error::other(e.to_string()))); let mut builder = Response::builder().status(status); for name in [ header::CONTENT_TYPE, @@ -135,7 +134,7 @@ pub async fn receive_pack( metrics::counter!("walgit_push_forwarded_total", "outcome" => outcome).increment(1); tracing::info!( status = status.as_u16(), - elapsed_ms = started.elapsed().as_millis() as u64, + elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), "push broker response streamed" ); ForwardOutcome::Response(output) diff --git a/crates/walgit-server/src/instance.rs b/crates/walgit-server/src/instance.rs index a8872c5..0eb39db 100644 --- a/crates/walgit-server/src/instance.rs +++ b/crates/walgit-server/src/instance.rs @@ -58,14 +58,12 @@ fn cgroup_cpus() -> Option { // cgroup v2: "quota period"; v1: cpu.cfs_quota_us / cpu.cfs_period_us. if let Ok(s) = std::fs::read_to_string("/sys/fs/cgroup/cpu.max") { let mut it = s.split_whitespace(); - if let (Some(q), Some(p)) = (it.next(), it.next()) { - if q != "max" { - if let (Ok(q), Ok(p)) = (q.parse::(), p.parse::()) { - if p > 0.0 { - return Some((q / p).round().max(1.0) as usize); - } - } - } + if let (Some(q), Some(p)) = (it.next(), it.next()) + && q != "max" + && let (Ok(q), Ok(p)) = (q.parse::(), p.parse::()) + && p > 0.0 + { + return Some((q / p).round().max(1.0) as usize); } } None @@ -92,7 +90,7 @@ fn gce_machine_type() -> Option { let s = String::from_utf8_lossy(&out.stdout).trim().to_string(); s.rsplit('/') .next() - .map(|m| m.to_string()) + .map(std::string::ToString::to_string) .filter(|m| !m.is_empty()) }) .clone() @@ -100,9 +98,9 @@ fn gce_machine_type() -> Option { fn gib(b: u64) -> String { let g = b as f64 / (1u64 << 30) as f64; if g >= 10.0 { - format!("{:.0} GiB", g) + format!("{g:.0} GiB") } else { - format!("{:.1} GiB", g) + format!("{g:.1} GiB") } } @@ -125,8 +123,9 @@ pub fn info(cfg: &walgit_config::Config) -> InstanceInfo { .or_else(|| env("HOSTNAME")) .unwrap_or_else(|| "walgit".into()); let revision = env("WALGIT_REVISION").unwrap_or_default(); - let instance = env("WALGIT_INSTANCE_ID") - .map(|i| { + let instance = env("WALGIT_INSTANCE_ID").map_or_else( + || std::process::id().to_string(), + |i| { i.chars() .rev() .take(6) @@ -134,8 +133,8 @@ pub fn info(cfg: &walgit_config::Config) -> InstanceInfo { .chars() .rev() .collect() - }) - .unwrap_or_else(|| std::process::id().to_string()); + }, + ); let version = match option_env!("WALGIT_BUILD_SHA") { Some(sha) if !sha.is_empty() => format!( "{}+{}", @@ -153,16 +152,14 @@ pub fn info(cfg: &walgit_config::Config) -> InstanceInfo { .map(|r| format!("{r:?}").to_lowercase()) .collect() }; - let cpus = cgroup_cpus().unwrap_or_else(|| { - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1) - }); + let cpus = cgroup_cpus() + .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, std::num::NonZero::get)); let memory_bytes = cgroup_memory_max().or_else(meminfo_total).unwrap_or(0); let shape = match kind { - "ssd" => gce_machine_type() - .map(|m| format!("{m} · {cpus} vCPU · {}", gib(memory_bytes))) - .unwrap_or_else(|| format!("{cpus} vCPU · {}", gib(memory_bytes))), + "ssd" => gce_machine_type().map_or_else( + || format!("{cpus} vCPU · {}", gib(memory_bytes)), + |m| format!("{m} · {cpus} vCPU · {}", gib(memory_bytes)), + ), "serverless" => format!("a serverless host · {cpus} vCPU · {}", gib(memory_bytes)), _ => format!("{cpus} cpus · {}", gib(memory_bytes)), }; diff --git a/crates/walgit-server/src/lfs.rs b/crates/walgit-server/src/lfs.rs index d6b3f29..4a12e18 100644 --- a/crates/walgit-server/src/lfs.rs +++ b/crates/walgit-server/src/lfs.rs @@ -1,5 +1,7 @@ //! Git LFS batch API + basic transfer (download/upload/verify). Objects live at //! `lfs/objects///` in the repo-scoped store. +use std::collections::HashMap; + use axum::body::Body; use axum::http::{HeaderMap, StatusCode}; use axum::response::{IntoResponse, Response}; @@ -109,7 +111,7 @@ pub async fn batch( .batch(upstream, cfg.upstream.token_env.as_deref(), &missing) .await } - _ => Default::default(), + _ => HashMap::default(), }; let mut objs = Vec::with_capacity(body.objects.len()); @@ -209,7 +211,7 @@ pub async fn batch( } /// `GET|HEAD /{repo}/info/lfs/objects/{oid}` — stream the object with the full -/// immutable-object contract (strong ETag, 304, Range/If-Range, HEAD, +/// immutable-object contract (strong `ETag`, 304, Range/If-Range, HEAD, /// Content-Length); see `static_object`. LFS objects are sha256-addressed. pub async fn get_object( st: &AppState, @@ -387,6 +389,10 @@ pub async fn put_object( headers: &HeaderMap, body: Body, ) -> Result { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + use sha2::{Digest, Sha256}; + if !st.cfg.lfs.enabled { return Err(ApiError::NotFound("lfs disabled".into())); } @@ -405,8 +411,6 @@ pub async fn put_object( .map_err(|e| ApiError::Internal(e.to_string()))?, ); let mut reader = body_to_async_read(body); - use sha2::{Digest, Sha256}; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; let mut hasher = Sha256::new(); let mut n = 0u64; let mut buf = vec![0u8; 64 * 1024]; diff --git a/crates/walgit-server/src/lfs_upstream.rs b/crates/walgit-server/src/lfs_upstream.rs index f058321..886f6a2 100644 --- a/crates/walgit-server/src/lfs_upstream.rs +++ b/crates/walgit-server/src/lfs_upstream.rs @@ -110,7 +110,7 @@ impl Upstream { match result { Ok(m) => m, Err(error) => { - tracing::warn!(%error, elapsed_ms = started.elapsed().as_millis() as u64, "lfs upstream batch failed; treating as absent"); + tracing::warn!(%error, elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), "lfs upstream batch failed; treating as absent"); HashMap::new() } } diff --git a/crates/walgit-server/src/lib.rs b/crates/walgit-server/src/lib.rs index 8df7c6e..4b139e5 100644 --- a/crates/walgit-server/src/lib.rs +++ b/crates/walgit-server/src/lib.rs @@ -1,5 +1,34 @@ //! Git smart HTTP server (protocol v0/v2), LFS, bundle serving, admin, health, metrics. //! See AGENTS.md Phase 3. +#![allow( + clippy::case_sensitive_file_extension_comparisons, + clippy::cast_possible_truncation, + clippy::cast_possible_wrap, + clippy::cast_precision_loss, + clippy::cast_sign_loss, + clippy::doc_lazy_continuation, + clippy::expect_used, + clippy::if_same_then_else, + clippy::implicit_hasher, + clippy::indexing_slicing, + clippy::many_single_char_names, + clippy::match_wildcard_for_single_variants, + clippy::needless_continue, + clippy::needless_pass_by_value, + clippy::ref_option, + clippy::string_slice, + clippy::struct_field_names, + clippy::too_many_arguments, + clippy::trivially_copy_pass_by_ref, + clippy::type_complexity, + clippy::unnested_or_patterns, + clippy::unnecessary_wraps, + clippy::unused_async, + clippy::unused_self, + clippy::unwrap_used, + clippy::unreadable_literal, + clippy::used_underscore_binding +)] pub mod admin; pub mod auth; @@ -77,7 +106,7 @@ pub struct AppState { } impl AppState { - /// Build a full AppState from a config + store (memory or opened backend). + /// Build a full `AppState` from a config + store (memory or opened backend). pub async fn new( cfg: Arc, store: DynStore, @@ -143,7 +172,8 @@ pub fn router(state: Arc) -> Router { state.clone(), web::require_auth, )); - let inner = Router::new() + + Router::new() .merge( web::api::router(state.clone()) .with_state(()) @@ -176,7 +206,7 @@ pub fn router(state: Arc) -> Router { body: Body| async move { bridge::http_notify(&st, &headers, body) .await - .unwrap_or_else(|e| e.into_response()) + .unwrap_or_else(axum::response::IntoResponse::into_response) }, ), ) @@ -222,17 +252,15 @@ pub fn router(state: Arc) -> Router { state.inflight.clone(), middleware::request_id, )) - .with_state(state); - inner + .with_state(state) } async fn host_from_authority(mut req: Request) -> Request { - if !req.headers().contains_key(axum::http::header::HOST) { - if let Some(auth) = req.uri().authority().map(|a| a.to_string()) { - if let Ok(v) = axum::http::HeaderValue::from_str(&auth) { - req.headers_mut().insert(axum::http::header::HOST, v); - } - } + if !req.headers().contains_key(axum::http::header::HOST) + && let Some(auth) = req.uri().authority().map(std::string::ToString::to_string) + && let Ok(v) = axum::http::HeaderValue::from_str(&auth) + { + req.headers_mut().insert(axum::http::header::HOST, v); } req } @@ -241,7 +269,10 @@ fn panic_response(err: Box) -> Response { let msg = err .downcast_ref::() .cloned() - .or_else(|| err.downcast_ref::<&str>().map(|s| s.to_string())) + .or_else(|| { + err.downcast_ref::<&str>() + .map(std::string::ToString::to_string) + }) .unwrap_or_else(|| "unknown panic".to_string()); tracing::error!(panic = %msg, "request handler panicked"); ( @@ -285,7 +316,7 @@ fn spawn_runtime_watchdog( }) .map(|pages| pages * 4096 / (1024 * 1024)); tracing::warn!( - gap_ms = gap.as_millis() as u64, + gap_ms = u64::try_from(gap.as_millis()).unwrap_or(u64::MAX), inflight, tasks_running, lock_wait_max_ms = walgit_wal::lockwait::max_wait_ms(), @@ -507,7 +538,7 @@ impl axum::serve::Listener for NodelayListener { } } -/// Enable TCP_NODELAY on an accepted stream. Applied via `Listener::tap_io` so +/// Enable `TCP_NODELAY` on an accepted stream. Applied via `Listener::tap_io` so /// the connection stays a plain `TcpStream` and axum's blanket `Connected` impl /// for `TapIo` supplies the peer `SocketAddr` to `ConnectInfo` (used by the /// accel-redirect loopback check). Git's receive-pack status is many small @@ -572,27 +603,24 @@ pub async fn serve( tokio::time::sleep(std::time::Duration::from_secs(2)).await; }; let serving = async move { - match tls { - Some(t) => { - axum::serve( - tls::TlsListener { - tcp: listener, - acceptor: t.acceptor.clone(), - }, - app, - ) - .with_graceful_shutdown(graceful) - .await - } - None => { - use axum::serve::ListenerExt; - axum::serve( - NodelayListener(listener).tap_io(set_nodelay), - app.into_make_service_with_connect_info::(), - ) - .with_graceful_shutdown(graceful) - .await - } + if let Some(t) = tls { + axum::serve( + tls::TlsListener { + tcp: listener, + acceptor: t.acceptor.clone(), + }, + app, + ) + .with_graceful_shutdown(graceful) + .await + } else { + use axum::serve::ListenerExt; + axum::serve( + NodelayListener(listener).tap_io(set_nodelay), + app.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(graceful) + .await } }; // In-flight requests get `server.drain_timeout` from phase 2 on (a stuck @@ -601,7 +629,7 @@ pub async fn serve( let bound = state_for_shutdown.cfg.server.drain_timeout; tokio::select! { r = serving => r?, - _ = async { phase2.notified().await; tokio::time::sleep(bound).await } => { + () = async { phase2.notified().await; tokio::time::sleep(bound).await } => { tracing::warn!(?bound, "shutdown: in-flight requests still open past server.drain_timeout; exiting"); } } @@ -700,18 +728,14 @@ impl walgit_bundle::BundleSource for RegistryBundleSource { }; } Err(e) => { - tracing::warn!(repo = %id, error = %e, "remote reader unavailable for bundle build; using git") + tracing::warn!(repo = %id, error = %e, "remote reader unavailable for bundle build; using git"); } } } - let linked = h - .local() - .packs() - .map(|ps| { - ps.iter() - .any(|p| h.local().pack_path(&p.checksum).is_symlink()) - }) - .unwrap_or(false); + let linked = h.local().packs().is_ok_and(|ps| { + ps.iter() + .any(|p| h.local().pack_path(&p.checksum).is_symlink()) + }); if linked { return walgit_bundle::BundleEngine::Gix { faulter: None }; } @@ -755,7 +779,7 @@ mod listen_tests { .await .unwrap(); let port = m.local_addr().unwrap().port(); - if !m.addrs().iter().any(|a| a.is_ipv6()) { + if !m.addrs().iter().any(std::net::SocketAddr::is_ipv6) { return; // no IPv6 on this host } tokio::net::TcpStream::connect((std::net::Ipv6Addr::LOCALHOST, port)) diff --git a/crates/walgit-server/src/maintain.rs b/crates/walgit-server/src/maintain.rs index 9f12a2f..7f619bf 100644 --- a/crates/walgit-server/src/maintain.rs +++ b/crates/walgit-server/src/maintain.rs @@ -55,7 +55,7 @@ pub async fn run_loop(state: Arc) { let ticker = { let (state, host, last_unit) = (state.clone(), host.clone(), last_unit.clone()); tokio::spawn(async move { - let mut t = tokio::time::interval(std::time::Duration::from_secs(120)); + let mut t = tokio::time::interval(std::time::Duration::from_mins(2)); t.tick().await; loop { t.tick().await; @@ -70,7 +70,7 @@ pub async fn run_loop(state: Arc) { match outcome { Ok(r) => { if let Some(u) = &r.last_unit { - last_unit = u.clone(); + last_unit.clone_from(u); } span.record("repos", r.repos); span.record("units", r.units); @@ -89,7 +89,7 @@ pub async fn run_loop(state: Arc) { } Err(e) => { span.record("outcome", "error"); - warn!(error = %e, "maintenance pass failed") + warn!(error = %e, "maintenance pass failed"); } } metrics::histogram!("walgit_maintain_pass_seconds", "host" => host.clone()) @@ -251,7 +251,7 @@ pub async fn next_unit(state: &Arc, id: &RepoId) -> anyhow::Result id.to_string()) .set(m.head_seq.saturating_sub(cp_seq) as f64); if let Some(t) = m @@ -268,10 +268,10 @@ pub async fn next_unit(state: &Arc, id: &RepoId) -> anyhow::Result, id: &RepoId) -> anyhow::Result 0 => tracing::info!(repo = %id, pruned = n, "bundle retention applied"), Ok(_) => {} Err(e) => { - tracing::warn!(repo = %id, error = %e, "bundle retention failed; the next publish applies it") + tracing::warn!(repo = %id, error = %e, "bundle retention failed; the next publish applies it"); } } match state @@ -307,11 +307,11 @@ pub async fn next_unit(state: &Arc, id: &RepoId) -> anyhow::Result 0 => { - tracing::info!(repo = %id, settled = n, "closed bundle slots settled") + tracing::info!(repo = %id, settled = n, "closed bundle slots settled"); } Ok(_) => {} Err(e) => { - tracing::warn!(repo = %id, error = %e, "settling closed slots failed; units will measure them") + tracing::warn!(repo = %id, error = %e, "settling closed slots failed; units will measure them"); } } let rows = state @@ -371,10 +371,9 @@ pub async fn next_unit(state: &Arc, id: &RepoId) -> anyhow::Result, id: &RepoId) -> anyhow::Result { let at = f.at.as_ref() - .map(walgit_proto::time::to_system) - .unwrap_or(SystemTime::UNIX_EPOCH); + .map_or(SystemTime::UNIX_EPOCH, walgit_proto::time::to_system); let age = SystemTime::now().duration_since(at).unwrap_or_default(); (age >= interval).then(|| format!("last audit {}h ago", age.as_secs() / 3600)) } @@ -505,8 +503,7 @@ pub async fn upcoming( }; let slot = next .duration_since(SystemTime::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); + .map_or(0, |d| d.as_secs()); let (unit, host) = match strat.kind { walgit_config::BundleKind::Full => match &base { Some(b) if many || base_predates_window(handle, strat, slot, b.seq).await => { @@ -620,7 +617,7 @@ pub async fn run_pass(state: &Arc) -> anyhow::Result { if value .as_ref() .and_then(|v| v.get("built")) - .and_then(|b| b.as_bool()) + .and_then(serde_json::Value::as_bool) .unwrap_or(false) { report.bundles += 1; @@ -688,7 +685,7 @@ pub async fn run_pass(state: &Arc) -> anyhow::Result { } /// Heartbeats older than this are a departed host, not a stale one. -const HEARTBEAT_EXPIRY: std::time::Duration = std::time::Duration::from_secs(24 * 3600); +const HEARTBEAT_EXPIRY: std::time::Duration = std::time::Duration::from_hours(24); /// Every maintainer heartbeat in the bucket (expired ones purged). pub async fn heartbeats( @@ -700,24 +697,24 @@ pub async fn heartbeats( let mut keys = state.store.list(walgit_proto::keys::MAINTAIN_DIR, None); while let Some(m) = keys.next().await { let m = m?; - if let Some((meta, bytes)) = state.store.get_bytes(&m.key).await? { - if let Ok(hb) = walgit_proto::v1::MaintainerHeartbeat::decode(bytes.as_ref()) { - // A host that has not passed for a day is gone: purge its - // heartbeat so the plan shows only live maintainers. - let age = hb - .last_pass_at - .as_ref() - .map(walgit_proto::time::to_system) - .and_then(|t| SystemTime::now().duration_since(t).ok()); - if age.is_some_and(|a| a > HEARTBEAT_EXPIRY) { - if state.cfg.has_role(walgit_config::Role::Maintain) { - info!(host = %hb.host, age_secs = age.map(|a| a.as_secs()).unwrap_or(0), "maintenance: purging expired heartbeat"); - let _ = state.store.delete(&m.key, Some(meta.version)).await; - } - continue; + if let Some((meta, bytes)) = state.store.get_bytes(&m.key).await? + && let Ok(hb) = walgit_proto::v1::MaintainerHeartbeat::decode(bytes.as_ref()) + { + // A host that has not passed for a day is gone: purge its + // heartbeat so the plan shows only live maintainers. + let age = hb + .last_pass_at + .as_ref() + .map(walgit_proto::time::to_system) + .and_then(|t| SystemTime::now().duration_since(t).ok()); + if age.is_some_and(|a| a > HEARTBEAT_EXPIRY) { + if state.cfg.has_role(walgit_config::Role::Maintain) { + info!(host = %hb.host, age_secs = age.map_or(0, |a| a.as_secs()), "maintenance: purging expired heartbeat"); + let _ = state.store.delete(&m.key, Some(meta.version)).await; } - out.push(hb); + continue; } + out.push(hb); } } Ok(out) @@ -784,8 +781,7 @@ async fn run_op_value( ) -> Option { let started = Instant::now(); let task = match crate::ops::start(state.clone(), id.clone(), op, params).await { - Ok(t) => t, - Err(crate::ops::StartError::AlreadyRunning(t)) => t, + Ok(t) | Err(crate::ops::StartError::AlreadyRunning(t)) => t, Err(crate::ops::StartError::UnknownOp) => { warn!(repo = %id, op, "maintenance: cannot start op"); return None; @@ -793,13 +789,13 @@ async fn run_op_value( }; // Bounded: a maintenance op that runs longer than an hour is reported and // left running (it stays discoverable at …/tasks); the pass moves on. - if !task.wait_done(std::time::Duration::from_secs(3600)).await { + if !task.wait_done(std::time::Duration::from_hours(1)).await { warn!(repo = %id, op, "maintenance: op still running after 1h; moving on"); return None; } match task.outcome() { Some(Ok(o)) => { - info!(repo = %id, op, ms = started.elapsed().as_millis() as u64, "maintenance: done"); + info!(repo = %id, op, ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), "maintenance: done"); Some(o.value.unwrap_or(serde_json::Value::Null)) } Some(Err((_, msg))) => { diff --git a/crates/walgit-server/src/metrics.rs b/crates/walgit-server/src/metrics.rs index 96ef46a..264809e 100644 --- a/crates/walgit-server/src/metrics.rs +++ b/crates/walgit-server/src/metrics.rs @@ -14,10 +14,12 @@ static HANDLE: OnceLock> = OnceLock::new(); /// Install the Prometheus recorder once per process and return a shared handle. /// Safe to call repeatedly (subsequent calls return the same handle). pub fn install() -> anyhow::Result> { + use metrics_exporter_prometheus::PrometheusBuilder; + if let Some(h) = HANDLE.get() { return Ok(h.clone()); } - use metrics_exporter_prometheus::PrometheusBuilder; + let rec = PrometheusBuilder::new().build_recorder(); let handle = Arc::new(rec.handle()); // set_global_recorder fails if already set; ignore that race — the handle is diff --git a/crates/walgit-server/src/middleware.rs b/crates/walgit-server/src/middleware.rs index 6b41893..78de0d7 100644 --- a/crates/walgit-server/src/middleware.rs +++ b/crates/walgit-server/src/middleware.rs @@ -86,8 +86,10 @@ pub async fn request_id( .get(REQUEST_ID_HEADER) .and_then(|v| v.to_str().ok()) .filter(|s| !s.is_empty()) - .map(|s| s.to_string()) - .unwrap_or_else(|| Uuid::new_v4().to_string()); + .map_or_else( + || Uuid::new_v4().to_string(), + std::string::ToString::to_string, + ); if let Ok(hv) = HeaderValue::from_str(&id) { req.headers_mut().insert(REQUEST_ID_HEADER, hv); } diff --git a/crates/walgit-server/src/ops.rs b/crates/walgit-server/src/ops.rs index 3dbe9b2..659cb90 100644 --- a/crates/walgit-server/src/ops.rs +++ b/crates/walgit-server/src/ops.rs @@ -201,8 +201,7 @@ pub async fn read_fsck( fn flag(params: &HashMap, key: &str) -> bool { params .get(key) - .map(|v| matches!(v.as_str(), "1" | "true" | "yes" | "on")) - .unwrap_or(false) + .is_some_and(|v| matches!(v.as_str(), "1" | "true" | "yes" | "on")) } async fn run( @@ -276,7 +275,7 @@ async fn run( .map_err(|e| format!("writing fsck.pb: {e}"))?; metrics::gauge!("walgit_repo_missing_objects", "repo" => id.to_string()) .set(missing.len() as f64); - tracing::info!(repo = %id, seq, missing = missing.len(), problems = report.problems, elapsed_ms = t0.elapsed().as_millis() as u64, "fsck recorded"); + tracing::info!(repo = %id, seq, missing = missing.len(), problems = report.problems, elapsed_ms = u64::try_from(t0.elapsed().as_millis()).unwrap_or(u64::MAX), "fsck recorded"); let summary = if report.ok { format!( "fsck clean ({lines} lines, {:.0}s)", @@ -378,7 +377,7 @@ async fn run( .map_err(|e| format!("writing fsck.pb: {e}"))?; metrics::counter!("walgit_repair_objects_total", "repo" => id.to_string()) .increment(pack.objects); - tracing::info!(repo = %id, seq, objects = pack.objects, bytes = pack.bytes, %upstream, elapsed_ms = t0.elapsed().as_millis() as u64, "repair published"); + tracing::info!(repo = %id, seq, objects = pack.objects, bytes = pack.bytes, %upstream, elapsed_ms = u64::try_from(t0.elapsed().as_millis()).unwrap_or(u64::MAX), "repair published"); Ok(( format!( "repaired {} object(s) ({} bytes) from upstream at seq {seq}", @@ -402,7 +401,7 @@ async fn run( .write_rev_index(&oid) .await .map_err(|e| format!("rev-index: {e}"))?; - let bytes = std::fs::metadata(&rev).map(|m| m.len()).unwrap_or(0); + let bytes = std::fs::metadata(&rev).map_or(0, |m| m.len()); log(format!( "pack-{checksum}.rev: {bytes} bytes in {:.1}s; publishing", t0.elapsed().as_secs_f64() @@ -411,7 +410,7 @@ async fn run( .annotate_pack(&checksum, Some(rev), None, None) .await .map_err(|e| format!("rev-index publish: {e}"))?; - tracing::info!(repo = %id, pack = %checksum, bytes, elapsed_ms = t0.elapsed().as_millis() as u64, "rev index published"); + tracing::info!(repo = %id, pack = %checksum, bytes, elapsed_ms = u64::try_from(t0.elapsed().as_millis()).unwrap_or(u64::MAX), "rev index published"); Ok(( format!("pack-{checksum}.rev ({bytes} bytes) published"), serde_json::json!({"pack": checksum, "bytes": bytes}), @@ -445,8 +444,7 @@ async fn run( "building {strategy} slot {slot} ({})", walgit_bundle::slots::from_epoch(slot) .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0) + .map_or(0, |d| d.as_secs()) )); // A FULL slot of a repository that has a tier-2 base is a compose // of that base (header = refs at the base's seq) — never a diff --git a/crates/walgit-server/src/pktline.rs b/crates/walgit-server/src/pktline.rs index 54830f8..015b053 100644 --- a/crates/walgit-server/src/pktline.rs +++ b/crates/walgit-server/src/pktline.rs @@ -8,9 +8,11 @@ pub const MAX_DATA_LEN: usize = 65516; /// Encode a data line into `buf`. Panics if `data` exceeds [`MAX_DATA_LEN`]. pub fn encode_line(buf: &mut Vec, data: &[u8]) { + const HEX: &[u8; 16] = b"0123456789abcdef"; + assert!(data.len() <= MAX_DATA_LEN, "pkt-line too long"); let len = data.len() + 4; - const HEX: &[u8; 16] = b"0123456789abcdef"; + buf.extend_from_slice(&[ HEX[(len >> 12) & 0xf], HEX[(len >> 8) & 0xf], diff --git a/crates/walgit-server/src/policy.rs b/crates/walgit-server/src/policy.rs index 59a608f..33a3dd7 100644 --- a/crates/walgit-server/src/policy.rs +++ b/crates/walgit-server/src/policy.rs @@ -183,9 +183,9 @@ impl RepoPolicy { if !rule_names.insert(&r.name) { return Err(format!("rules: duplicate name {:?}", r.name)); } - let n = r.effect.protect.is_some() as u8 - + r.effect.history.is_some() as u8 - + r.effect.size.is_some() as u8; + let n = u8::from(r.effect.protect.is_some()) + + u8::from(r.effect.history.is_some()) + + u8::from(r.effect.size.is_some()); if n != 1 { return Err(format!( "rule {:?}: effect must have exactly one of protect, history, size", @@ -246,8 +246,8 @@ fn check_overlap_bypass(p: &RepoPolicy) -> Result<(), String> { if ba.is_empty() || bb.is_empty() { continue; } - let set_a: HashSet<&str> = ba.iter().map(|s| s.as_str()).collect(); - let set_b: HashSet<&str> = bb.iter().map(|s| s.as_str()).collect(); + let set_a: HashSet<&str> = ba.iter().map(std::string::String::as_str).collect(); + let set_b: HashSet<&str> = bb.iter().map(std::string::String::as_str).collect(); if set_a.is_disjoint(&set_b) { return Err(format!( "protect rules {:?} and {:?} overlap with disjoint bypass lists", diff --git a/crates/walgit-server/src/prewarm.rs b/crates/walgit-server/src/prewarm.rs index cf14bf5..854b02b 100644 --- a/crates/walgit-server/src/prewarm.rs +++ b/crates/walgit-server/src/prewarm.rs @@ -5,6 +5,7 @@ //! (discoverable at `…/tasks`); `/readyz` can be gated on completion //! (`cache.prewarm_ready_timeout`). +use std::collections::HashMap; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Instant; @@ -67,8 +68,8 @@ pub fn spawn(state: Arc) { let _p = sem.acquire().await; let t = Instant::now(); match warm(&st, &r).await { - Ok(summary) => tracing::info!(repo = %r, elapsed_ms = t.elapsed().as_millis() as u64, "prewarm: {summary}"), - Err(e) => tracing::warn!(repo = %r, elapsed_ms = t.elapsed().as_millis() as u64, "prewarm failed: {e}"), + Ok(summary) => tracing::info!(repo = %r, elapsed_ms = u64::try_from(t.elapsed().as_millis()).unwrap_or(u64::MAX), "prewarm: {summary}"), + Err(e) => tracing::warn!(repo = %r, elapsed_ms = u64::try_from(t.elapsed().as_millis()).unwrap_or(u64::MAX), "prewarm failed: {e}"), } st.readiness.pending.fetch_sub(1, Ordering::AcqRel); })); @@ -86,7 +87,7 @@ async fn warm(st: &Arc, repo: &str) -> Result { .parse() .map_err(|e: walgit_git::GitError| e.to_string())?; let handle = st.registry.open(&id).await.map_err(|e| e.to_string())?; - let task = match handle.begin_task("prewarm", Default::default()) { + let task = match handle.begin_task("prewarm", HashMap::default()) { walgit_wal::Begin::Started(t) => t, walgit_wal::Begin::AlreadyRunning(_) => return Ok("already warming".into()), }; @@ -142,20 +143,19 @@ async fn warm(st: &Arc, repo: &str) -> Result { .find(|r| r.name == head.head_target) .map(|r| r.oid.clone()); if let (Some(sha), walgit_wal::ObjectAccess::Remote(packs)) = (head_sha.as_deref(), &access) + && let Ok(oid) = gix_hash::ObjectId::from_hex(sha.as_bytes()) { - if let Ok(oid) = gix_hash::ObjectId::from_hex(sha.as_bytes()) { - reporter.notice(format!( - "Reading the root tree of {} from the pack set", - &sha[..12] - )); - let remote = crate::web::objects::Remote::new( - packs.clone(), - handle.local().clone(), - reporter.clone(), - ); - let (_c, tree, _m) = remote.fault_path(&oid, "").await.map_err(|e| e.message())?; - let _ = remote.tree_entries(&tree).await; - } + reporter.notice(format!( + "Reading the root tree of {} from the pack set", + &sha[..12] + )); + let remote = crate::web::objects::Remote::new( + packs.clone(), + handle.local().clone(), + reporter.clone(), + ); + let (_c, tree, _m) = remote.fault_path(&oid, "").await.map_err(|e| e.message())?; + let _ = remote.tree_entries(&tree).await; } Ok(format!("warm: {mode}")) } diff --git a/crates/walgit-server/src/rebuild.rs b/crates/walgit-server/src/rebuild.rs index 13fd769..560951e 100644 --- a/crates/walgit-server/src/rebuild.rs +++ b/crates/walgit-server/src/rebuild.rs @@ -129,8 +129,13 @@ fn disk_avail(path: &Path) -> Option { use std::ffi::CString; use std::os::unix::ffi::OsStrExt; let c = CString::new(path.as_os_str().as_bytes()).ok()?; + // SAFETY: statvfs is a C integer struct; all-zero is a valid initialized value. + #[allow(unsafe_code)] let mut st: libc::statvfs = unsafe { std::mem::zeroed() }; - if unsafe { libc::statvfs(c.as_ptr(), &mut st) } != 0 { + // SAFETY: c is NUL-terminated and live; st is aligned writable storage for statvfs. + #[allow(unsafe_code)] + let result = unsafe { libc::statvfs(c.as_ptr(), &raw mut st) }; + if result != 0 { return None; } Some(st.f_bavail as u64 * st.f_frsize as u64) @@ -348,7 +353,7 @@ pub async fn rebuild_base( if let Some(p) = already && p.tier == 2 && (p.has_bitmap || info.history_of.is_some()) - && supersedes_left.as_ref().is_none_or(|s| s.is_empty()) + && supersedes_left.as_ref().is_none_or(std::vec::Vec::is_empty) { log(format!( "pack {hex} is already live as tier 2: not re-published" diff --git a/crates/walgit-server/src/settings.rs b/crates/walgit-server/src/settings.rs index 735fd74..c25b98d 100644 --- a/crates/walgit-server/src/settings.rs +++ b/crates/walgit-server/src/settings.rs @@ -403,7 +403,7 @@ pub async fn http_validate( Ok(eff) => { let preview = walgit_proto::v1::RepoSettings { toml: text.to_string(), - revision: h.settings().map(|s| s.revision + 1).unwrap_or(1), + revision: h.settings().map_or(1, |s| s.revision + 1), author: "(preview)".into(), updated_at: None, message: String::new(), @@ -467,7 +467,7 @@ pub async fn http_policy_dry_run( .unwrap_or(20) .clamp(1, 200); let bytes = crate::collect_body(body).await?; - let policy = if bytes.iter().all(|b| b.is_ascii_whitespace()) { + let policy = if bytes.iter().all(u8::is_ascii_whitespace) { crate::policy::load(&st.store, &route.id) .await .map_err(|e| ApiError::Internal(e.to_string()))? diff --git a/crates/walgit-server/src/smart.rs b/crates/walgit-server/src/smart.rs index 583e231..da59a86 100644 --- a/crates/walgit-server/src/smart.rs +++ b/crates/walgit-server/src/smart.rs @@ -1,8 +1,8 @@ //! Git smart HTTP protocol (v0/v2): info/refs, upload-pack, receive-pack. //! //! References: -//! * https://git-scm.com/docs/http-protocol -//! * https://git-scm.com/docs/protocol-v2 +//! * +//! * use std::collections::HashMap; use std::sync::Arc; @@ -66,10 +66,8 @@ pub async fn info_refs( } return Err(auth_err(e)); } - if is_receive { - if let Some(msg) = push_url_must_be_git(st, route, headers) { - return Ok(git_err_response("git-receive-pack", &msg)); - } + if is_receive && let Some(msg) = push_url_must_be_git(st, route, headers) { + return Ok(git_err_response("git-receive-pack", &msg)); } let service = match service_param.as_str() { @@ -92,31 +90,28 @@ pub async fn info_refs( pktline::encode_text(&mut buf, &svc_line); pktline::encode_flush(&mut buf); - match (protocol, service) { - (walgit_git::pkt::Protocol::V2, walgit_git::Service::UploadPack) => { - v2_capability_advert(st, &route.id, &handle, &mut buf).await?; - } - _ => { - // v0 (and receive-pack always). - let repo_key = route.id.to_string(); - let ver = handle.manifest_version(); - if let Some(cached) = st - .caches + if let (walgit_git::pkt::Protocol::V2, walgit_git::Service::UploadPack) = (protocol, service) { + v2_capability_advert(st, &route.id, &handle, &mut buf).await?; + } else { + // v0 (and receive-pack always). + let repo_key = route.id.to_string(); + let ver = handle.manifest_version(); + if let Some(cached) = st + .caches + .ref_advert + .get_v0(&repo_key, ver.as_ref(), service) + { + buf.extend_from_slice(&cached); + } else { + let start = buf.len(); + handle + .local() + .advertise_refs_v0(service, &mut buf) + .map_err(git_err)?; + let advert_bytes = buf[start..].to_vec(); + st.caches .ref_advert - .get_v0(&repo_key, ver.as_ref(), service) - { - buf.extend_from_slice(&cached); - } else { - let start = buf.len(); - handle - .local() - .advertise_refs_v0(service, &mut buf) - .map_err(git_err)?; - let advert_bytes = buf[start..].to_vec(); - st.caches - .ref_advert - .insert_v0(&repo_key, ver.as_ref(), service, advert_bytes); - } + .insert_v0(&repo_key, ver.as_ref(), service, advert_bytes); } } @@ -126,10 +121,10 @@ pub async fn info_refs( fn parse_query(query: &str, key: &str) -> Option { for pair in query.split('&') { - if let Some((k, v)) = pair.split_once('=') { - if k == key { - return Some(v.to_string()); - } + if let Some((k, v)) = pair.split_once('=') + && k == key + { + return Some(v.to_string()); } } None @@ -162,10 +157,10 @@ async fn v2_capability_advert( walgit_git::ObjectFormat::Sha256 => "sha256", }; pktline::encode_text(buf, &format!("object-format={fmt}\n")); - if st.cfg.bundles.advertise { - if let Ok(Some(_list)) = st.bundles.list(id).await { - pktline::encode_text(buf, "bundle-uri\n"); - } + if st.cfg.bundles.advertise + && let Ok(Some(_list)) = st.bundles.list(id).await + { + pktline::encode_text(buf, "bundle-uri\n"); } pktline::encode_flush(buf); Ok(()) @@ -224,24 +219,22 @@ async fn upload_pack_v2( }; let repo_key = route.id.to_string(); let version = handle.manifest_version(); - let lines = - match st - .caches + let lines = if let Some(lines) = + st.caches .ref_advert .get_v2_ls_refs(&repo_key, version.as_ref(), &args) - { - Some(lines) => lines, - None => { - let lines = handle.local().ls_refs(&args).map_err(git_err)?; - st.caches.ref_advert.insert_v2_ls_refs( - &repo_key, - version.as_ref(), - &args, - lines.clone(), - ); - lines - } - }; + { + lines + } else { + let lines = handle.local().ls_refs(&args).map_err(git_err)?; + st.caches.ref_advert.insert_v2_ls_refs( + &repo_key, + version.as_ref(), + &args, + lines.clone(), + ); + lines + }; let mut buf = Vec::with_capacity(1024); for line in &lines { pktline::encode_text(&mut buf, &line.render(&args)); @@ -297,35 +290,32 @@ async fn upload_pack_v2( // list within the hour TRIED bundle-uri — its zero-have fetch is a // bundle download that failed (git never retries one). Let that // clone succeed through upload-pack, once per 6 h, loudly. - match bundle_fallback_allowed(st, headers, route).await { - Some(who) => { - tracing::warn!(repo = %route.id, principal = %who, "bundles.require: one-shot upload-pack fallback for a client whose bundle download failed"); - metrics::counter!("walgit_bundle_fallback_total", "repo" => route.id.to_string()).increment(1); - fallback_warning = Some(format!( - "walgit: WARNING — your git fetched the bundle list but could not apply the bundles \ - (a bundle download failed or was cut; see the warnings above). Serving this clone's \ - full history through upload-pack ONCE (≈ 32 GB for acme/monorepo, minutes of server \ - time); the next such clone within 6 h is refused. Faster next time: retry the clone \ - (bundle downloads are cached at the edge), or the blobless form: \ - git clone --filter=blob:none --bundle-uri={base}/{repo}.git/bundles/list?filter=blob:none {base}/{repo}.git", - base = request_base_url(st, headers), - repo = route.id - )); - } - None => { - let msg = bundles_required_message(st, headers, route); - return Ok(if req.sideband_all { - let mut buf = sideband_pkt(3, &msg); - pktline::encode_flush(&mut buf); - text_response( - "application/x-git-upload-pack-result", - no_cache_headers(), - buf, - ) - } else { - git_err_response("git-upload-pack", &msg) - }); - } + if let Some(who) = bundle_fallback_allowed(st, headers, route).await { + tracing::warn!(repo = %route.id, principal = %who, "bundles.require: one-shot upload-pack fallback for a client whose bundle download failed"); + metrics::counter!("walgit_bundle_fallback_total", "repo" => route.id.to_string()).increment(1); + fallback_warning = Some(format!( + "walgit: WARNING — your git fetched the bundle list but could not apply the bundles \ + (a bundle download failed or was cut; see the warnings above). Serving this clone's \ + full history through upload-pack ONCE (≈ 32 GB for acme/monorepo, minutes of server \ + time); the next such clone within 6 h is refused. Faster next time: retry the clone \ + (bundle downloads are cached at the edge), or the blobless form: \ + git clone --filter=blob:none --bundle-uri={base}/{repo}.git/bundles/list?filter=blob:none {base}/{repo}.git", + base = request_base_url(st, headers), + repo = route.id + )); + } else { + let msg = bundles_required_message(st, headers, route); + return Ok(if req.sideband_all { + let mut buf = sideband_pkt(3, &msg); + pktline::encode_flush(&mut buf); + text_response( + "application/x-git-upload-pack-result", + no_cache_headers(), + buf, + ) + } else { + git_err_response("git-upload-pack", &msg) + }); } } // Narrated fetch: the client accepted sideband-all and wants @@ -381,8 +371,7 @@ async fn upload_pack_v2( let size = gix_hash::ObjectId::from_hex(hex.as_bytes()) .ok() .and_then(|oid| repo.find_object(oid).ok()) - .map(|o| o.data.len() as i64) - .unwrap_or(-1); + .map_or(-1, |o| o.data.len() as i64); pktline::encode_text(&mut sizes_buf, &format!("size {size}\n")); } pktline::encode_flush(&mut sizes_buf); @@ -394,7 +383,7 @@ async fn upload_pack_v2( } "bundle-uri" => { let _guard = handle.sync_refs().await.map_err(wal_err)?; - let _ = walgit_git::pkt::parse_bundle_uri(&cmd); + let () = walgit_git::pkt::parse_bundle_uri(&cmd); let base = request_base_url(st, headers); let lines = st .bundles @@ -478,7 +467,7 @@ fn bundle_narration( out.push("bundle-uri: none of your haves is a bundle tip — your git did not use the bundles (clone with the recipe from the Clone menu, or check transfer.bundleURI)".into()); } else { let bytes: u64 = applied.iter().map(|b| b.size).sum(); - let newest = applied.last().map(|b| b.creation_token).unwrap_or(0); + let newest = applied.last().map_or(0, |b| b.creation_token); let when = chrono::DateTime::from_timestamp(newest as i64, 0) .map(|d| d.format("%Y-%m-%d %H:%MZ").to_string()) .unwrap_or_default(); @@ -527,7 +516,7 @@ async fn run_fetch( bytes = stats.bytes, faulted, rounds, - ms = t0.elapsed().as_millis() as u64, + ms = u64::try_from(t0.elapsed().as_millis()).unwrap_or(u64::MAX), "gix fetch over remote-served base" ); return Ok(()); @@ -583,7 +572,9 @@ async fn sync_narrated<'h, W: tokio::io::AsyncWrite + Unpin>( } let sync = handle.sync(); tokio::pin!(sync); - let mut last_bar = std::time::Instant::now() - std::time::Duration::from_secs(1); + let mut last_bar = std::time::Instant::now() + .checked_sub(std::time::Duration::from_secs(1)) + .unwrap(); loop { tokio::select! { biased; @@ -591,7 +582,7 @@ async fn sync_narrated<'h, W: tokio::io::AsyncWrite + Unpin>( p = rx.recv() => match p { Ok(walgit_wal::Progress::Notice { text }) => { let _ = say(writer, &text).await; } Ok(walgit_wal::Progress::Progress { label, done, total, unit, percent }) => { - if last_bar.elapsed() >= std::time::Duration::from_secs(1) || total.map(|t| done >= t).unwrap_or(false) { + if last_bar.elapsed() >= std::time::Duration::from_secs(1) || total.is_some_and(|t| done >= t) { last_bar = std::time::Instant::now(); let line = match (total, percent) { (Some(t), Some(pc)) if unit == "bytes" => format!("{label}: {pc:.0}% ({} / {})", human(done), human(t)), @@ -613,7 +604,7 @@ async fn sync_narrated<'h, W: tokio::io::AsyncWrite + Unpin>( break (&mut sync).await; } }, - _ = tokio::time::sleep(std::time::Duration::from_secs(5)) => { + () = tokio::time::sleep(std::time::Duration::from_secs(5)) => { let _ = say(writer, &format!("still syncing ({}s)…", t0.elapsed().as_secs())).await; } } @@ -640,8 +631,7 @@ async fn narrated_fetch( .require_read(headers) .await .ok() - .map(|p| p.name) - .unwrap_or_else(|| "anonymous".into()); + .map_or_else(|| "anonymous".into(), |p| p.name); // Nothing that can wait (store reads, syncs) happens before the stream // is open and the first band-2 line is out: the bundle facts are read // inside the task, after the greeting. @@ -742,7 +732,7 @@ async fn narrated_fetch( } }; let local = guard.local().clone(); - let packs = local.packs().map(|p| p.len()).unwrap_or(0); + let packs = local.packs().map_or(0, |p| p.len()); let remote = handle.remote_served(); let _ = say( &mut writer, @@ -1070,7 +1060,7 @@ pub async fn receive_pack( .get("x-request-id") .and_then(|v| v.to_str().ok()) .filter(|v| !v.is_empty()) - .map(|v| v.to_string()); + .map(std::string::ToString::to_string); if !caps.side_band_64k { // No sideband: the response is the report alone, after the work. @@ -1189,31 +1179,30 @@ async fn receive_pack_process( }; // Connectivity check for pushed tips (before we publish anything). - if unpack_err.is_none() && st.cfg.wal.check_connectivity { - if let Ok(Some(_)) = &ingest { - let tips: Vec = txn - .updates - .iter() - .filter(|u| !u.new_oid.is_empty() && !is_zero_oid(&u.new_oid)) - .filter_map(|u| gix_hash::ObjectId::from_hex(u.new_oid.as_bytes()).ok()) - .collect(); - if !tips.is_empty() { - if let Err(e) = local - .check_connectivity_async(&tips, true) - .instrument(tracing::info_span!( - "receive.connectivity", - tips = tips.len() - )) - .await - { - // Every refusal names the reason on each ref: `unpack ng` - // alone makes git print "remote failed to report status". - tracing::warn!(repo = %route_id, error = %e, "receive-pack: connectivity check failed"); - metrics::counter!("walgit_push_refused_total", "reason" => "connectivity") - .increment(1); - return Ok(refusal_report(&caps, &txn, &format!("connectivity: {e}")).await); - } - } + if unpack_err.is_none() + && st.cfg.wal.check_connectivity + && let Ok(Some(_)) = &ingest + { + let tips: Vec = txn + .updates + .iter() + .filter(|u| !u.new_oid.is_empty() && !is_zero_oid(&u.new_oid)) + .filter_map(|u| gix_hash::ObjectId::from_hex(u.new_oid.as_bytes()).ok()) + .collect(); + if !tips.is_empty() + && let Err(e) = local + .check_connectivity_async(&tips, true) + .instrument(tracing::info_span!( + "receive.connectivity", + tips = tips.len() + )) + .await + { + // Every refusal names the reason on each ref: `unpack ng` + // alone makes git print "remote failed to report status". + tracing::warn!(repo = %route_id, error = %e, "receive-pack: connectivity check failed"); + metrics::counter!("walgit_push_refused_total", "reason" => "connectivity").increment(1); + return Ok(refusal_report(&caps, &txn, &format!("connectivity: {e}")).await); } } @@ -1416,9 +1405,8 @@ async fn parse_fetch_request( .map_err(git_err)?; match line { None - | Some(walgit_git::pkt::PktLine::Flush) - | Some(walgit_git::pkt::PktLine::Delim) => break, - Some(walgit_git::pkt::PktLine::ResponseEnd) => break, + | Some(walgit_git::pkt::PktLine::Flush | walgit_git::pkt::PktLine::Delim) + | Some(walgit_git::pkt::PktLine::ResponseEnd) => break, Some(walgit_git::pkt::PktLine::Data(b)) => { let s = String::from_utf8_lossy(&b); let s = s.trim_end_matches('\n'); @@ -1621,9 +1609,9 @@ fn too_large_message( /// How often one principal may fall back to an upload-pack full clone of a /// `bundles.require` repository. -const FALLBACK_EVERY: std::time::Duration = std::time::Duration::from_secs(6 * 3600); +const FALLBACK_EVERY: std::time::Duration = std::time::Duration::from_hours(6); /// How recent the principal's `bundles/list` fetch must be to count as "tried". -const ATTEMPT_WINDOW: std::time::Duration = std::time::Duration::from_secs(3600); +const ATTEMPT_WINDOW: std::time::Duration = std::time::Duration::from_hours(1); /// D17 amendment: `Some(principal)` when this zero-have full fetch may go to /// upload-pack — the principal fetched the repo's bundle list within the hour diff --git a/crates/walgit-server/src/sse.rs b/crates/walgit-server/src/sse.rs index 015fd80..b5b7d60 100644 --- a/crates/walgit-server/src/sse.rs +++ b/crates/walgit-server/src/sse.rs @@ -106,14 +106,13 @@ impl Rendered { etag, } } - /// Plain HTTP response (honours `If-None-Match` when an ETag is set). + /// Plain HTTP response (honours `If-None-Match` when an `ETag` is set). pub fn into_response(self, req: &HeaderMap) -> Response { if let Some(etag) = &self.etag { let hit = req .get(header::IF_NONE_MATCH) .and_then(|v| v.to_str().ok()) - .map(|v| v.split(',').any(|t| t.trim() == etag || t.trim() == "*")) - .unwrap_or(false); + .is_some_and(|v| v.split(',').any(|t| t.trim() == etag || t.trim() == "*")); if hit { let mut r = StatusCode::NOT_MODIFIED.into_response(); r.headers_mut().insert(header::ETAG, etag.parse().unwrap()); @@ -235,7 +234,7 @@ pub fn task_stream(state: std::sync::Arc) -> Respo _ = done.changed() => { if *done.borrow() { break; } } - _ = tokio::time::sleep(KEEPALIVE) => { + () = tokio::time::sleep(KEEPALIVE) => { if tx.send(Bytes::from_static(b": keepalive\n\n")).await.is_err() { return; } } } diff --git a/crates/walgit-server/src/static_object.rs b/crates/walgit-server/src/static_object.rs index 7548c00..2c714f5 100644 --- a/crates/walgit-server/src/static_object.rs +++ b/crates/walgit-server/src/static_object.rs @@ -1,9 +1,9 @@ //! HTTP serving of immutable store objects (bundles, LFS objects, packs) with //! the complete conditional/range contract a CDN or `git` expects: //! -//! * strong `ETag` = the store version (GCS generation / S3 ETag), quoted; +//! * strong `ETag` = the store version (GCS generation / S3 `ETag`), quoted; //! * `If-None-Match` (list or `*`) → `304` with the same validators; -//! * `If-Range` (ETag or ignored date) gating `Range`; +//! * `If-Range` (`ETag` or ignored date) gating `Range`; //! * single byte ranges incl. open-ended (`bytes=N-`) and suffix (`bytes=-N`), //! `206` + `Content-Range`, `416` + `Content-Range: bytes */total`; //! * `HEAD` answered from metadata (no body download); @@ -136,7 +136,7 @@ fn if_none_match_hit(headers: &HeaderMap, version: &Version) -> bool { tags.iter().any(|t| t == "*" || t == cur) } -/// `If-Range`: if it names an ETag that does not match the current version the +/// `If-Range`: if it names an `ETag` that does not match the current version the /// range is ignored and the full body is sent (RFC 9110 §13.1.5). Dates are /// not supported (we have no `Last-Modified`) and therefore also ignored. fn if_range_allows(headers: &HeaderMap, version: &Version) -> bool { @@ -189,13 +189,13 @@ fn base_headers(resp: &mut Response, meta: &ObjectMeta, opts: &ServeOptions<'_>) h.insert(header::CACHE_CONTROL, cache_control(opts)); h.insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes")); h.insert(header::VARY, HeaderValue::from_static("Accept-Encoding")); - if let Some(name) = opts.filename { - if let Ok(v) = HeaderValue::from_str(&format!( + if let Some(name) = opts.filename + && let Ok(v) = HeaderValue::from_str(&format!( "attachment; filename=\"{}\"", name.replace('"', "") - )) { - h.insert(header::CONTENT_DISPOSITION, v); - } + )) + { + h.insert(header::CONTENT_DISPOSITION, v); } } @@ -247,44 +247,42 @@ pub async fn serve( && !head && accel_requested(headers) && opts.peer.is_some_and(|p| p.ip().is_loopback()) + && let Some(target) = store.accel_target(key).await { - if let Some(target) = store.accel_target(key).await { - let meta = match store.head(key).await { - Ok(Some(m)) => m, - Ok(None) => return Err(ApiError::NotFound(format!("{key} not found"))), - Err(e) => return Err(e.into()), - }; - if if_none_match_hit(headers, &meta.version) { - return Ok(not_modified(&meta.version, &opts)); - } - let mut resp = StatusCode::OK.into_response(); - base_headers(&mut resp, &meta, &opts); - let h = resp.headers_mut(); - let hv = |s: &str| { - HeaderValue::from_str(s) - .map_err(|e| ApiError::Internal(format!("accel header: {e}"))) - }; - h.insert("x-accel-redirect", HeaderValue::from_static(ACCEL_LOCATION)); - // Where and how the edge fetches. nginx keeps the upstream headers of this answer - // across the internal redirect and never forwards them to the client. - h.insert("x-walgit-store-url", hv(&target.url)?); - if let Some(auth) = &target.authorization { - h.insert("x-walgit-store-authorization", hv(auth)?); - } - // The edge's cache key: the object, not the (possibly presigned, changing) URL. - h.insert( - "x-walgit-store-key", - hv(&walgit_store::util::encode_path(key))?, - ); - h.insert("x-walgit-accel", HeaderValue::from_static(store.backend())); - // nginx keeps only Content-Type/Disposition, Accept-Ranges, Cache-Control and Expires - // of this answer across the internal redirect and would otherwise hand the client - // the bucket's ETag (md5/crc form) — different from the version ETag our HEAD/304 use, - // so `If-Range` would fail and a resumed download get the whole object. The edge - // re-emits this header as the response ETag and hides the bucket's. - h.insert("x-walgit-etag", etag_of(&meta.version)); - return Ok(resp); + let meta = match store.head(key).await { + Ok(Some(m)) => m, + Ok(None) => return Err(ApiError::NotFound(format!("{key} not found"))), + Err(e) => return Err(e.into()), + }; + if if_none_match_hit(headers, &meta.version) { + return Ok(not_modified(&meta.version, &opts)); + } + let mut resp = StatusCode::OK.into_response(); + base_headers(&mut resp, &meta, &opts); + let h = resp.headers_mut(); + let hv = |s: &str| { + HeaderValue::from_str(s).map_err(|e| ApiError::Internal(format!("accel header: {e}"))) + }; + h.insert("x-accel-redirect", HeaderValue::from_static(ACCEL_LOCATION)); + // Where and how the edge fetches. nginx keeps the upstream headers of this answer + // across the internal redirect and never forwards them to the client. + h.insert("x-walgit-store-url", hv(&target.url)?); + if let Some(auth) = &target.authorization { + h.insert("x-walgit-store-authorization", hv(auth)?); } + // The edge's cache key: the object, not the (possibly presigned, changing) URL. + h.insert( + "x-walgit-store-key", + hv(&walgit_store::util::encode_path(key))?, + ); + h.insert("x-walgit-accel", HeaderValue::from_static(store.backend())); + // nginx keeps only Content-Type/Disposition, Accept-Ranges, Cache-Control and Expires + // of this answer across the internal redirect and would otherwise hand the client + // the bucket's ETag (md5/crc form) — different from the version ETag our HEAD/304 use, + // so `If-Range` would fail and a resumed download get the whole object. The edge + // re-emits this header as the response ETag and hides the bucket's. + h.insert("x-walgit-etag", etag_of(&meta.version)); + return Ok(resp); } // HEAD and Range both need the size before deciding what to fetch. For diff --git a/crates/walgit-server/src/stream.rs b/crates/walgit-server/src/stream.rs index 835be53..7dc976a 100644 --- a/crates/walgit-server/src/stream.rs +++ b/crates/walgit-server/src/stream.rs @@ -12,11 +12,11 @@ use futures::stream::StreamExt; use tokio::io::{AsyncRead, AsyncWrite}; use tokio_util::io::{ReaderStream, StreamReader}; -/// Convert an axum request body into an `AsyncRead`. Map errors to io::Error. +/// Convert an axum request body into an `AsyncRead`. Map errors to `io::Error`. pub fn body_to_async_read(body: Body) -> impl AsyncRead + Unpin + Send { let stream = body .into_data_stream() - .map(|res| res.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))); + .map(|res| res.map_err(|e| io::Error::other(e.to_string()))); StreamReader::new(stream) } @@ -56,6 +56,12 @@ pub fn write_body_pipe(buf: usize) -> (tokio::io::DuplexStream, Body) { /// small pkt-line responses (report-status, ls-refs) into a buffer. pub struct VecWriter(pub Vec); +impl Default for VecWriter { + fn default() -> Self { + Self::new() + } +} + impl VecWriter { pub fn new() -> Self { Self(Vec::new()) diff --git a/crates/walgit-server/src/telemetry.rs b/crates/walgit-server/src/telemetry.rs index 216ad8c..59ad1bc 100644 --- a/crates/walgit-server/src/telemetry.rs +++ b/crates/walgit-server/src/telemetry.rs @@ -57,10 +57,10 @@ fn resolve_project_id(cfg: &Config) -> Option { if let Some(p) = &cfg.telemetry.trace_project { return Some(p.clone()); } - if let Ok(p) = std::env::var("GOOGLE_CLOUD_PROJECT") { - if !p.is_empty() { - return Some(p); - } + if let Ok(p) = std::env::var("GOOGLE_CLOUD_PROJECT") + && !p.is_empty() + { + return Some(p); } // Probe metadata only when the documented GCE override is present. Off-GCP, // resolving metadata.google.internal can otherwise stall startup. @@ -246,7 +246,7 @@ where .values .get("trace_id") .and_then(|v| v.as_str()) - .map(|s| s.to_string()); + .map(std::string::ToString::to_string); let parent_trace = ctx .span(id) .and_then(|s| s.parent()) @@ -298,8 +298,10 @@ where .values .get("message") .and_then(|v| v.as_str()) - .map(|s| s.to_string()) - .unwrap_or_else(|| metadata.name().to_string()); + .map_or_else( + || metadata.name().to_string(), + std::string::ToString::to_string, + ); let mut record = self.base_record(severity, &message); record.insert("target".into(), json!(metadata.target())); @@ -325,8 +327,7 @@ where if let Some(t) = visitor.values.get("trace_id").and_then(|v| v.as_str()) { let sid = trace .as_ref() - .map(|t| t.1.clone()) - .unwrap_or_else(generate_span_id); + .map_or_else(generate_span_id, |t| t.1.clone()); trace = Some((t.to_string(), sid)); } if let Some((tid, sid)) = trace { @@ -348,12 +349,13 @@ where return; } let end = data.last_exit.unwrap_or_else(Instant::now); - let elapsed_ms = end.duration_since(data.start).as_millis() as u64; + let elapsed_ms = + u64::try_from(end.duration_since(data.start).as_millis()).unwrap_or(u64::MAX); record = self.base_record(level_to_severity(&data.level), data.name); record.insert("elapsed_ms".into(), json!(elapsed_ms)); // Close deferred well past the last poll (a lingering child): say so // separately instead of inflating the work's duration. - let idle_ms = end.elapsed().as_millis() as u64; + let idle_ms = u64::try_from(end.elapsed().as_millis()).unwrap_or(u64::MAX); if idle_ms >= 1000 { record.insert("close_deferred_ms".into(), json!(idle_ms)); } @@ -382,12 +384,11 @@ where // --------------------------------------------------------------------------- fn level_to_severity(level: &Level) -> &'static str { - match level { - &Level::ERROR => "ERROR", - &Level::WARN => "WARNING", - &Level::INFO => "INFO", - &Level::DEBUG => "DEBUG", - &Level::TRACE => "DEBUG", + match *level { + Level::ERROR => "ERROR", + Level::WARN => "WARNING", + Level::INFO => "INFO", + Level::DEBUG | Level::TRACE => "DEBUG", } } @@ -410,7 +411,7 @@ struct FieldCollector { impl Visit for FieldCollector { fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { - let s = format!("{:?}", value); + let s = format!("{value:?}"); self.values.insert(field.name().to_string(), json!(s)); } @@ -446,7 +447,7 @@ impl Visit for FieldCollector { /// Parse the `X-Cloud-Trace-Context` header. /// Format: `TRACE_ID/SPAN_ID;o=TRACE_TRUE` -/// Returns the trace_id (32-char hex). +/// Returns the `trace_id` (32-char hex). pub fn parse_x_cloud_trace_context(header: &str) -> Option { let trace_id = header.split('/').next()?; let trimmed = trace_id.trim(); @@ -460,7 +461,7 @@ pub fn parse_x_cloud_trace_context(header: &str) -> Option { /// Parse the W3C `traceparent` header. /// Format: `00-TRACE_ID-PARENT_ID-TRACE_FLAGS` -/// Returns the trace_id (32-char hex). +/// Returns the `trace_id` (32-char hex). pub fn parse_traceparent(header: &str) -> Option { let parts: Vec<&str> = header.split('-').collect(); if parts.len() >= 4 { @@ -478,15 +479,14 @@ pub fn extract_trace_id(headers: &axum::http::HeaderMap) -> Option { if let Some(v) = headers .get("x-cloud-trace-context") .and_then(|v| v.to_str().ok()) + && let Some(tid) = parse_x_cloud_trace_context(v) { - if let Some(tid) = parse_x_cloud_trace_context(v) { - return Some(tid); - } + return Some(tid); } - if let Some(v) = headers.get("traceparent").and_then(|v| v.to_str().ok()) { - if let Some(tid) = parse_traceparent(v) { - return Some(tid); - } + if let Some(v) = headers.get("traceparent").and_then(|v| v.to_str().ok()) + && let Some(tid) = parse_traceparent(v) + { + return Some(tid); } None } @@ -500,7 +500,7 @@ static PROJECT_ID: OnceLock> = OnceLock::new(); /// Initialise `tracing-subscriber` from `[telemetry]`. /// /// * `log_format` selects JSON (Cloud Logging) or pretty (human). -/// * `log_filter` is the default EnvFilter; `RUST_LOG` overrides it entirely. +/// * `log_filter` is the default `EnvFilter`; `RUST_LOG` overrides it entirely. /// * When JSON, installs a [`CloudLoggingLayer`] that emits structured JSON /// with Cloud Logging trace correlation and span-close performance lines. pub fn tracing_init(cfg: &Config) { @@ -577,7 +577,7 @@ mod tests { assert_eq!(CloudLoggingLayer::span_kind("noprefix"), "other"); } - /// Verify that extract_trace_id works with both header formats. + /// Verify that `extract_trace_id` works with both header formats. #[test] fn extract_trace_id_from_headers() { let mut headers = axum::http::HeaderMap::new(); diff --git a/crates/walgit-server/src/tls.rs b/crates/walgit-server/src/tls.rs index 7114d72..ae84df6 100644 --- a/crates/walgit-server/src/tls.rs +++ b/crates/walgit-server/src/tls.rs @@ -106,7 +106,7 @@ fn self_signed(dir: &Path, hostnames: &[String]) -> anyhow::Result<(String, Stri rcgen::CertificateParams::new(hostnames.to_vec()).context("certificate params")?; params.distinguished_name.push( rcgen::DnType::CommonName, - hostnames.first().map(String::as_str).unwrap_or("walgit"), + hostnames.first().map_or("walgit", String::as_str), ); params.not_before = rcgen::date_time_ymd(2024, 1, 1); params.not_after = rcgen::date_time_ymd(2124, 1, 1); @@ -137,7 +137,7 @@ fn write_private(path: &Path, body: &str) -> anyhow::Result<()> { } /// `axum::serve::Listener` that wraps every accepted TCP connection in a -/// lazily-handshaking TLS stream (TCP_NODELAY set, like the plain listener). +/// lazily-handshaking TLS stream (`TCP_NODELAY` set, like the plain listener). pub struct TlsListener { pub(crate) tcp: TcpAccept, pub acceptor: TlsAcceptor, diff --git a/crates/walgit-server/src/web/api.rs b/crates/walgit-server/src/web/api.rs index 61d2203..2d26704 100644 --- a/crates/walgit-server/src/web/api.rs +++ b/crates/walgit-server/src/web/api.rs @@ -86,7 +86,11 @@ impl From for Commit { let (body, trailers) = super::trailers::split_trailers(&m.body); Commit { sha: m.id.to_string(), - parents: m.parents.iter().map(|p| p.to_string()).collect(), + parents: m + .parents + .iter() + .map(std::string::ToString::to_string) + .collect(), author: m.author, author_email: m.author_email, author_date: m.author_date, @@ -239,7 +243,7 @@ pub struct Repo { pub(crate) index: Arc, handle: Arc, access: ObjectAccess, - /// Whether objects are readable (Need::Objects satisfied). + /// Whether objects are readable (`Need::Objects` satisfied). objects: bool, reporter: Reporter, /// Shared render cache (object store) — set for remotely served repos. @@ -375,18 +379,17 @@ where metrics::counter!("walgit_api_immutable_hit", "tier" => "memory").increment(1); return Ok(Rendered::json(hit, IMMUTABLE, None).into_response(headers)); } - if slow && st.cfg.cache.shared_render_cache { - if let Ok(walgit_store::GetResult::Object { body, meta }) = handle + if slow + && st.cfg.cache.shared_render_cache + && let Ok(walgit_store::GetResult::Object { body, meta }) = handle .store() .get(&shared_key(key), GetOptions::default()) .await - { - if let Ok(b) = walgit_store::util::collect(body, meta.size as usize).await { - metrics::counter!("walgit_api_immutable_hit", "tier" => "store").increment(1); - st.caches.api_immutable.insert(key.clone(), b.clone()); - return Ok(Rendered::json(b, IMMUTABLE, None).into_response(headers)); - } - } + && let Ok(b) = walgit_store::util::collect(body, meta.size as usize).await + { + metrics::counter!("walgit_api_immutable_hit", "tier" => "store").increment(1); + st.caches.api_immutable.insert(key.clone(), b.clone()); + return Ok(Rendered::json(b, IMMUTABLE, None).into_response(headers)); } } if slow && crate::sse::wants_sse(headers) { @@ -511,7 +514,7 @@ async fn refs( None, |r| async move { let head = r.index.head().map(|(name, sha)| RefInfo { name, sha }); - let etag = etag_for(head.as_ref().map(|h| h.sha.as_str()).unwrap_or("unborn")); + let etag = etag_for(head.as_ref().map_or("unborn", |h| h.sha.as_str())); Ok(json_swr(&Refs { head }, Some(&etag))) }, ) @@ -542,7 +545,7 @@ async fn ref_list( let needle = q.q.as_deref() .filter(|s| !s.is_empty()) - .map(|s| s.to_ascii_lowercase()); + .map(str::to_ascii_lowercase); let after = q.after.as_deref().unwrap_or(""); // Byte-sorted: skip straight to the first candidate (> after, >= prefix). let lower = match &prefix { @@ -555,15 +558,15 @@ async fn ref_list( let mut refs = Vec::with_capacity(n.min(256)); let mut more = false; for (name, sha) in &list[start..] { - if let Some(p) = &prefix { - if !name.starts_with(p.as_str()) { - break; // sorted: no further names share the prefix - } + if let Some(p) = &prefix + && !name.starts_with(p.as_str()) + { + break; // sorted: no further names share the prefix } - if let Some(nd) = &needle { - if !name.to_ascii_lowercase().contains(nd.as_str()) { - continue; - } + if let Some(nd) = &needle + && !name.to_ascii_lowercase().contains(nd.as_str()) + { + continue; } if refs.len() == n { more = true; @@ -646,15 +649,15 @@ async fn resolve_rest(r: &Repo, rest: &str) -> Result { /// Resolve a single revision name (no path): branch, tag, then git rev-parse. async fn resolve_name(r: &Repo, name: &str) -> Result { - if name.is_empty() || name == "HEAD" { - if let Some((n, sha)) = r.index.head() { - return Ok(Resolved { - ref_name: n, - sha, - path: String::new(), - kind: "branch", - }); - } + if (name.is_empty() || name == "HEAD") + && let Some((n, sha)) = r.index.head() + { + return Ok(Resolved { + ref_name: n, + sha, + path: String::new(), + kind: "branch", + }); } if let Some(sha) = r.index.branch(name) { return Ok(Resolved { @@ -750,7 +753,7 @@ async fn resolve_impl( } /// Split `{ref}/{path}` for tree/blob: a leading full sha is taken verbatim -/// (immutable response); otherwise §3 resolution (SWR + ETag). +/// (immutable response); otherwise §3 resolution (SWR + `ETag`). fn split_addr(rest: &str) -> Option<(Resolved, bool)> { let rest = rest.trim_matches('/'); let (first, path) = match rest.split_once('/') { @@ -850,10 +853,8 @@ async fn tree( move |r| async move { let (res, immutable) = resolve_addr(&r, &rest).await?; let key = tree_key(&r.id, &res.sha, &res.path); - if immutable { - if let Some(hit) = st2.caches.api_immutable.get(&key) { - return Ok(Rendered::json(hit, IMMUTABLE, None)); - } + if immutable && let Some(hit) = st2.caches.api_immutable.get(&key) { + return Ok(Rendered::json(hit, IMMUTABLE, None)); } let body = match r.remote() { Some(remote) => render_tree_remote(&remote, &res).await?, @@ -914,16 +915,14 @@ async fn render_tree( .ok() .and_then(|b| parse_commits(&b).into_iter().next()); let mut readme = None; - if let Some(e) = readme_entry(&entries) { - if let Ok(content) = git(local, vec!["cat-file".into(), "blob".into(), e.sha.clone()]).await - { - if let Ok(s) = String::from_utf8(content) { - readme = Some(Readme { - name: e.name.clone(), - contents: s, - }); - } - } + if let Some(e) = readme_entry(&entries) + && let Ok(content) = git(local, vec!["cat-file".into(), "blob".into(), e.sha.clone()]).await + && let Ok(s) = String::from_utf8(content) + { + readme = Some(Readme { + name: e.name.clone(), + contents: s, + }); } Ok(json_bytes(&Tree { ref_name: res.ref_name.clone(), @@ -977,7 +976,7 @@ async fn render_tree_remote(remote: &Remote, res: &Resolved) -> Result = futures::stream::iter(raw.into_iter()) + let entries: Vec = futures::stream::iter(raw) .map(|e| async move { let kind = match e.mode.kind() { gix_object::tree::EntryKind::Tree => "tree", @@ -990,8 +989,7 @@ async fn render_tree_remote(remote: &Remote, res: &Resolved) -> Result Result>) = match r.remote() { - Some(remote) => { - let sha = gix_hash::ObjectId::from_hex(res.sha.as_bytes()) - .map_err(|_| not_found("revision"))?; - remote - .reporter - .notice(format!("Reading {} from the WAL pack set", res.path)); - let (_c, target, mode) = remote.fault_path(&sha, &res.path).await?; - if !mode.is_blob_or_symlink() { - return Err(not_found(format!("'{}' is not a file", res.path))); - } - let (_, size) = remote - .kind_and_size(&target) - .await? - .ok_or_else(|| not_found("blob"))?; - if size as usize > MAX_BLOB { - (size as i64, None) - } else { - let o = remote.get(&target).await?; - (size as i64, Some(o.data.to_vec())) - } + let (size, bytes): (i64, Option>) = if let Some(remote) = r.remote() { + let sha = gix_hash::ObjectId::from_hex(res.sha.as_bytes()) + .map_err(|_| not_found("revision"))?; + remote + .reporter + .notice(format!("Reading {} from the WAL pack set", res.path)); + let (_c, target, mode) = remote.fault_path(&sha, &res.path).await?; + if !mode.is_blob_or_symlink() { + return Err(not_found(format!("'{}' is not a file", res.path))); } - None => { - let bytes = git( - &r.local, - vec![ - "cat-file".into(), - "blob".into(), - format!("{}:{}", res.sha, res.path), - ], - ) - .await?; - (bytes.len() as i64, Some(bytes)) + let (_, size) = remote + .kind_and_size(&target) + .await? + .ok_or_else(|| not_found("blob"))?; + if size as usize > MAX_BLOB { + (size as i64, None) + } else { + let o = remote.get(&target).await?; + (size as i64, Some(o.data.to_vec())) } + } else { + let bytes = git( + &r.local, + vec![ + "cat-file".into(), + "blob".into(), + format!("{}:{}", res.sha, res.path), + ], + ) + .await?; + (bytes.len() as i64, Some(bytes)) }; let is_text = size <= MAX_BLOB as i64 && bytes .as_ref() - .map(|b| !b.contains(&0) && std::str::from_utf8(b).is_ok()) - .unwrap_or(false); + .is_some_and(|b| !b.contains(&0) && std::str::from_utf8(b).is_ok()); if raw && is_text { let etag = etag_for(&res.sha); return Ok(Rendered { @@ -1227,48 +1219,43 @@ async fn commits( (resolve_name(&r, &reference).await?, false) }; let key = commits_key(&r.id, &res.sha, &path, skip, n); - if immutable { - if let Some(hit) = st2.caches.api_immutable.get(&key) { - return Ok(Rendered::json(hit, IMMUTABLE, None)); - } + if immutable && let Some(hit) = st2.caches.api_immutable.get(&key) { + return Ok(Rendered::json(hit, IMMUTABLE, None)); } - let mut cs: Vec = match r.remote() { - Some(remote) => { - let start = gix_hash::ObjectId::from_hex(res.sha.as_bytes()) - .map_err(|_| not_found("revision"))?; - let label = if path.is_empty() { - "Walking history".to_string() - } else { - format!("Walking history of {path}") - }; - remote.reporter.notice(format!( - "{label} from {} (reading commits from the WAL pack set)", - &res.sha[..12] - )); - let all = remote - .walk( - start, - (!path.is_empty()).then_some(path.as_str()), - skip + n + 1, - &label, - ) - .await?; - all.into_iter().skip(skip).map(Commit::from).collect() - } - None => { - let mut a = vec![ - "log".into(), - format!("--format={}", log_format()), - "--no-color".into(), - format!("--skip={skip}"), - format!("-{count}", count = n.saturating_add(1)), - res.sha.clone(), - ]; - if !path.is_empty() { - a.extend(["--".into(), path.clone()]); - } - parse_commits(&git(&r.local, a).await?) + let mut cs: Vec = if let Some(remote) = r.remote() { + let start = gix_hash::ObjectId::from_hex(res.sha.as_bytes()) + .map_err(|_| not_found("revision"))?; + let label = if path.is_empty() { + "Walking history".to_string() + } else { + format!("Walking history of {path}") + }; + remote.reporter.notice(format!( + "{label} from {} (reading commits from the WAL pack set)", + &res.sha[..12] + )); + let all = remote + .walk( + start, + (!path.is_empty()).then_some(path.as_str()), + skip + n + 1, + &label, + ) + .await?; + all.into_iter().skip(skip).map(Commit::from).collect() + } else { + let mut a = vec![ + "log".into(), + format!("--format={}", log_format()), + "--no-color".into(), + format!("--skip={skip}"), + format!("-{count}", count = n.saturating_add(1)), + res.sha.clone(), + ]; + if !path.is_empty() { + a.extend(["--".into(), path.clone()]); } + parse_commits(&git(&r.local, a).await?) }; let more = cs.len() > n; if more { @@ -1314,10 +1301,8 @@ async fn commit_detail( resolve_name(&r, &rev).await?.sha }; let key = commit_key(&r.id, &sha); - if immutable { - if let Some(hit) = st2.caches.api_immutable.get(&key) { - return Ok(Rendered::json(hit, IMMUTABLE, None)); - } + if immutable && let Some(hit) = st2.caches.api_immutable.get(&key) { + return Ok(Rendered::json(hit, IMMUTABLE, None)); } if let Some(remote) = r.remote() { // Fault the commit, its first parent and every object the diff @@ -1420,16 +1405,16 @@ fn parse_stats(bytes: &[u8]) -> Vec { /// `git --numstat -M` prints renames as `old => new` or `prefix/{old => new}/suffix`; /// return the new path. fn normalize_rename(s: &str) -> String { - if let (Some(open), Some(close)) = (s.find('{'), s.rfind('}')) { - if open < close { - let inner = &s[open + 1..close]; - if let Some((_, new)) = inner.split_once(" => ") { - let mut out = String::with_capacity(s.len()); - out.push_str(&s[..open]); - out.push_str(new); - out.push_str(&s[close + 1..]); - return out.replace("//", "/"); - } + if let (Some(open), Some(close)) = (s.find('{'), s.rfind('}')) + && open < close + { + let inner = &s[open + 1..close]; + if let Some((_, new)) = inner.split_once(" => ") { + let mut out = String::with_capacity(s.len()); + out.push_str(&s[..open]); + out.push_str(new); + out.push_str(&s[close + 1..]); + return out.replace("//", "/"); } } if let Some((_, new)) = s.split_once(" => ") { diff --git a/crates/walgit-server/src/web/login.rs b/crates/walgit-server/src/web/login.rs index 08387c8..0cdcfc2 100644 --- a/crates/walgit-server/src/web/login.rs +++ b/crates/walgit-server/src/web/login.rs @@ -1,4 +1,4 @@ -//! Browser sign-in: the OpenID Connect authorization-code flow against +//! Browser sign-in: the `OpenID` Connect authorization-code flow against //! `server.auth.issuer`, done by walgit itself. `GET /_auth/login?next=/p` //! redirects to the issuer's `authorization_endpoint` (from discovery), //! `GET /_auth/callback` exchanges the code at the `token_endpoint`, verifies the @@ -71,7 +71,7 @@ fn loopback_origin(st: &AppState, headers: &HeaderMap) -> bool { let base = crate::smart::request_base_url(st, headers); let host = base.split("://").nth(1).unwrap_or(&base); let host = host.split('/').next().unwrap_or(host); - let host = host.rsplit_once(':').map(|(h, _)| h).unwrap_or(host); + let host = host.rsplit_once(':').map_or(host, |(h, _)| h); host == "walgit.localhost" || host == "localhost" || host == "127.0.0.1" || host == "[::1]" } @@ -116,8 +116,7 @@ fn walgit_origin(st: &AppState, headers: &HeaderMap) -> String { fn now() -> u64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0) + .map_or(0, |d| d.as_secs()) } fn urlencode(s: &str) -> String { @@ -125,9 +124,11 @@ fn urlencode(s: &str) -> String { for b in s.bytes() { match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { - out.push(b as char) + out.push(b as char); + } + _ => { + let _ = std::fmt::Write::write_fmt(&mut out, format_args!("%{b:02X}")); } - _ => out.push_str(&format!("%{b:02X}")), } } out @@ -147,15 +148,12 @@ async fn login( ) .into_response(); } - let disco = match st.auth.discovery().await { - Ok(d) => d, - Err(_) => { - return ( - StatusCode::SERVICE_UNAVAILABLE, - "identity provider unavailable (OIDC discovery failed)", - ) - .into_response(); - } + let Ok(disco) = st.auth.discovery().await else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "identity provider unavailable (OIDC discovery failed)", + ) + .into_response(); }; let (client_id, _) = st.auth.oauth_client().unwrap(); let next = safe_next(q.next); @@ -179,7 +177,9 @@ async fn login( ); // Google honours `hd` as a domain hint on its account chooser; other issuers ignore it. if let Some(hd) = st.cfg.server.auth.allowed_domains.first() { - url.push_str(&format!("&hd={}", urlencode(hd))); + { + let _ = std::fmt::Write::write_fmt(&mut url, format_args!("&hd={}", urlencode(hd))); + }; } let mut r = Redirect::to(&url).into_response(); r.headers_mut() diff --git a/crates/walgit-server/src/web/mod.rs b/crates/walgit-server/src/web/mod.rs index 21ad9b4..0d1979f 100644 --- a/crates/walgit-server/src/web/mod.rs +++ b/crates/walgit-server/src/web/mod.rs @@ -32,25 +32,26 @@ pub async fn canonical_browser_host( || path.starts_with("/services/public"); let browser = is_browser(req.headers()); let get = req.method() == axum::http::Method::GET || req.method() == axum::http::Method::HEAD; - if get && browser && !skip { - if let Some(dest) = walgit_localhost_host( + if get + && browser + && !skip + && let Some(dest) = walgit_localhost_host( req.headers() .get(header::HOST) .and_then(|v| v.to_str().ok()), - ) { - let scheme = if st.cfg.tls_enabled() { - "https" - } else { - "http" - }; - let pq = req - .uri() - .path_and_query() - .map(|p| p.as_str()) - .unwrap_or("/"); - let loc = format!("{scheme}://{dest}{pq}"); - return (StatusCode::FOUND, [(header::LOCATION, loc)]).into_response(); - } + ) + { + let scheme = if st.cfg.tls_enabled() { + "https" + } else { + "http" + }; + let pq = req + .uri() + .path_and_query() + .map_or("/", http::uri::PathAndQuery::as_str); + let loc = format!("{scheme}://{dest}{pq}"); + return (StatusCode::FOUND, [(header::LOCATION, loc)]).into_response(); } next.run(req).await } @@ -112,8 +113,7 @@ pub async fn require_auth( let next_url = req .uri() .path_and_query() - .map(|pq| pq.as_str().to_string()) - .unwrap_or_else(|| "/".to_string()); + .map_or_else(|| "/".to_string(), |pq| pq.as_str().to_string()); let q = url_encode(&next_url); return Redirect::temporary(&format!("/_auth/login?next={q}")).into_response(); } @@ -160,9 +160,11 @@ pub(crate) fn url_encode(s: &str) -> String { for b in s.bytes() { match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { - out.push(b as char) + out.push(b as char); + } + _ => { + let _ = std::fmt::Write::write_fmt(&mut out, format_args!("%{b:02X}")); } - _ => out.push_str(&format!("%{b:02X}")), } } out diff --git a/crates/walgit-server/src/web/objects.rs b/crates/walgit-server/src/web/objects.rs index aa150dd..73c819e 100644 --- a/crates/walgit-server/src/web/objects.rs +++ b/crates/walgit-server/src/web/objects.rs @@ -238,8 +238,7 @@ impl Remote { let entries = self.tree_entries(&cur).await?; let Some(e) = entries.into_iter().find(|e| e.name == seg.as_bytes()) else { return Err(not_found(format!( - "path '{}' does not exist in {}", - path, commit + "path '{path}' does not exist in {commit}" ))); }; cur = e.oid; @@ -248,8 +247,7 @@ impl Remote { self.fault(&cur).await?; } else if i + 1 < segs.len() { return Err(not_found(format!( - "path '{}' does not exist in {}", - path, commit + "path '{path}' does not exist in {commit}" ))); } else if e.mode.is_blob() { // blob: caller decides whether to fault (size check) @@ -349,7 +347,7 @@ impl Remote { .notice(format!("{label}: gave up after {budget} commits")); break; } - if popped % 100 == 0 { + if popped.is_multiple_of(100) { self.reporter .bar(label.to_string(), popped as u64, None, "commits"); } @@ -366,13 +364,12 @@ impl Remote { } else { let mut treesame_parent = None; for par in &meta.parents { - let pm = match metas.get(par) { - Some(m) => m.clone(), - None => { - let m = self.commit(par).await?; - metas.insert(*par, m.clone()); - m - } + let pm = if let Some(m) = metas.get(par) { + m.clone() + } else { + let m = self.commit(par).await?; + metas.insert(*par, m.clone()); + m }; let theirs = self.path_oid(&mut path_cache, pm.tree, p).await?; if theirs == mine { @@ -395,13 +392,12 @@ impl Remote { for par in follow { if seen.insert(par) { seq += 1; - let pm = match metas.get(&par) { - Some(m) => m.clone(), - None => { - let m = self.commit(&par).await?; - metas.insert(par, m.clone()); - m - } + let pm = if let Some(m) = metas.get(&par) { + m.clone() + } else { + let m = self.commit(&par).await?; + metas.insert(par, m.clone()); + m }; heap.push(Item(pm.commit_time, seq, par)); } diff --git a/crates/walgit-server/src/web/ui.rs b/crates/walgit-server/src/web/ui.rs index f34ccc6..4faaf0a 100644 --- a/crates/walgit-server/src/web/ui.rs +++ b/crates/walgit-server/src/web/ui.rs @@ -227,7 +227,7 @@ async fn setup_json( /// `GET|HEAD /repos.js` | `/repos.mjs` — the browser SDK (`web/sdk/`, built /// into `web/dist/` by `pnpm run build`). Permanent URL, so `no-cache` + -/// strong ETag (revalidated per deploy), precompressed like every asset. +/// strong `ETag` (revalidated per deploy), precompressed like every asset. pub async fn sdk_asset(req: Request) -> Response { let name = req.uri().path().trim_start_matches('/'); match embedded(name) { @@ -326,21 +326,21 @@ fn negotiate_encoding( .iter() .filter_map(|v| v.to_str().ok()) .flat_map(|v| v.split(',')) - .map(|t| t.trim()) + .map(str::trim) .filter(|t| !t.is_empty()) .collect::>(); let accepts = |name: &str| { accept.iter().any(|t| { let (coding, q) = t.split_once(';').map_or((*t, None), |(c, q)| (c, Some(q))); coding.trim().eq_ignore_ascii_case(name) - && !q.is_some_and(|q| q.trim().trim_start_matches("q=").trim() == "0") + && q.is_none_or(|q| q.trim().trim_start_matches("q=").trim() != "0") }) }; for (name, ext) in [("br", ".br"), ("gzip", ".gz")] { - if accepts(name) { - if let Some(f) = embedded(&format!("{path}{ext}")) { - return Some((Some(name), f.data)); - } + if accepts(name) + && let Some(f) = embedded(&format!("{path}{ext}")) + { + return Some((Some(name), f.data)); } } None @@ -349,12 +349,12 @@ fn negotiate_encoding( fn content_type(path: &str) -> &'static str { match Path::new(path).extension().and_then(|e| e.to_str()) { Some("css") => "text/css; charset=utf-8", - Some("js") | Some("mjs") => "text/javascript; charset=utf-8", - Some("json") | Some("map") => "application/json; charset=utf-8", + Some("js" | "mjs") => "text/javascript; charset=utf-8", + Some("json" | "map") => "application/json; charset=utf-8", Some("html") => "text/html; charset=utf-8", Some("svg") => "image/svg+xml", Some("png") => "image/png", - Some("jpg") | Some("jpeg") => "image/jpeg", + Some("jpg" | "jpeg") => "image/jpeg", Some("gif") => "image/gif", Some("webp") => "image/webp", Some("ico") => "image/x-icon", @@ -592,7 +592,7 @@ async fn overview( .map(|version| version.to_string()) .unwrap_or_default(); let base_url = crate::smart::request_base_url(&state, &headers); - let clone_url = format!("{}/{}.git", base_url, id); + let clone_url = format!("{base_url}/{id}.git"); let recipes = crate::setup::recipes(&state.cfg, &base_url, Some(&id.to_string())); let setup = recipes.setup_text.clone(); @@ -618,7 +618,7 @@ async fn overview( .iter() .filter(|entry| entry.kind() == EntryKind::Push) .filter_map(|entry| entry.created_at.as_ref().map(timestamp)) - .last(); + .next_back(); let mut push_count = 0; let mut compactions = Vec::new(); let mut pack_by_checksum = std::collections::HashMap::new(); @@ -738,9 +738,8 @@ async fn overview( "at the next `{w}` slot, on a maintainer whose capacity holds the pack set ({})", walgit_wal::remote::human_bytes(live_bytes) )), - (Some(_), false) => None, - (None, _) => None, - }, + (Some(_), false) | (None, _) => None, + }, }); } else if fresh >= ecfg.compaction.trigger_packs.max(2) { suggestions.push(Suggestion { @@ -756,7 +755,7 @@ async fn overview( }); } if manifest.head_seq > 0 { - let cp_seq = manifest.checkpoint.as_ref().map(|c| c.seq).unwrap_or(0); + let cp_seq = manifest.checkpoint.as_ref().map_or(0, |c| c.seq); let behind = manifest.head_seq.saturating_sub(cp_seq); if behind >= state.cfg.wal.snapshot_every_entries.max(1) || (cp_seq == 0 && behind > 0) { suggestions.push(Suggestion { @@ -952,7 +951,7 @@ async fn overview( disk: h.disk, max_pack_bytes: h.max_pack_bytes, last_pass_age_secs: age, - alive: age.map(|a| a < 600).unwrap_or(false), + alive: age.is_some_and(|a| a < 600), passes: h.passes, last_unit: h.last_unit, } @@ -1183,8 +1182,9 @@ async fn checkpoint_info( checkpoint.writer, ) } - Ok(GetResult::NotModified { .. }) => (0, String::new(), String::new()), - Err(walgit_store::StoreError::NotFound { .. }) => (0, String::new(), String::new()), + Ok(GetResult::NotModified { .. }) | Err(walgit_store::StoreError::NotFound { .. }) => { + (0, String::new(), String::new()) + } Err(error) => return Err(ApiError::Internal(error.to_string())), }; Ok(Some(BundleInfo { diff --git a/crates/walgit-server/src/web/v1.rs b/crates/walgit-server/src/web/v1.rs index 49813f8..137f371 100644 --- a/crates/walgit-server/src/web/v1.rs +++ b/crates/walgit-server/src/web/v1.rs @@ -329,7 +329,7 @@ struct RepoSummary { } /// `GET /{owner}/{repo}/api[-browser]` — one cheap, ref-level summary (SWR + -/// ETag on the head sha). Counts are O(1) from the ref index. +/// `ETag` on the head sha). Counts are O(1) from the ref index. async fn repo_summary( State(st): State>, headers: HeaderMap, @@ -346,7 +346,7 @@ async fn repo_summary( None, move |r| async move { let head = r.index.head().map(|(name, sha)| RefInfo { name, sha }); - let etag = etag_for(head.as_ref().map(|h| h.sha.as_str()).unwrap_or("unborn")); + let etag = etag_for(head.as_ref().map_or("unborn", |h| h.sha.as_str())); let full = format!("{o}/{n}"); Ok(json_swr( &RepoSummary { diff --git a/crates/walgit-server/tests/api_v1.rs b/crates/walgit-server/tests/api_v1.rs index db4e3af..266b2b3 100644 --- a/crates/walgit-server/tests/api_v1.rs +++ b/crates/walgit-server/tests/api_v1.rs @@ -1,3 +1,4 @@ +#![allow(clippy::many_single_char_names)] //! `/api/v1` (D20): the versioned programmatic surface, its browser-lane alias //! (`/api-browser`), CORS for foreign origins, discovery, `me`, repo summary and //! admin, and the SDK artefact route. diff --git a/crates/walgit-server/tests/drain.rs b/crates/walgit-server/tests/drain.rs index 03d648a..df1118e 100644 --- a/crates/walgit-server/tests/drain.rs +++ b/crates/walgit-server/tests/drain.rs @@ -9,6 +9,7 @@ mod harness; use harness::{Server, git, git_in}; +use std::collections::HashMap; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn after_sigterm_new_object_work_is_refused_and_no_unit_starts() -> anyhow::Result<()> { @@ -46,7 +47,7 @@ async fn after_sigterm_new_object_work_is_refused_and_no_unit_starts() -> anyhow .registry .open(&walgit_git::RepoId::new("o", "r")?) .await?; - let unit = match h0.begin_task("compact", Default::default()) { + let unit = match h0.begin_task("compact", HashMap::default()) { walgit_wal::Begin::Started(t) => t, walgit_wal::Begin::AlreadyRunning(_) => anyhow::bail!("compact already running"), }; diff --git a/crates/walgit-server/tests/e2e.rs b/crates/walgit-server/tests/e2e.rs index 849f645..7d165be 100644 --- a/crates/walgit-server/tests/e2e.rs +++ b/crates/walgit-server/tests/e2e.rs @@ -1,3 +1,7 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::unwrap_used)] +#![allow(clippy::many_single_char_names, unsafe_code)] + //! End-to-end tests: real upstream `git` against a live walgit-server backed //! by the in-memory store. Covers clone/push/fetch (v2 and v0), non-ff reject, //! ref delete, tags, partial clone + lazy fetch, ls-remote, and the two-instance @@ -712,7 +716,7 @@ async fn many_refs_impl(n: usize) -> TestResult { let push_start = Instant::now(); git_in(&src, &["push", "--mirror", "origin"])?; println!("{n}-ref mirror push took {:?}", push_start.elapsed()); - assert!(push_start.elapsed() < std::time::Duration::from_secs(240)); + assert!(push_start.elapsed() < std::time::Duration::from_mins(4)); let start = Instant::now(); let output = Command::new("git") .args(["ls-remote", &server.repo_url("t", "many-refs")]) @@ -981,8 +985,7 @@ fn git_lfs_present() -> bool { Command::new("git") .args(["lfs", "version"]) .output() - .map(|o| o.status.success()) - .unwrap_or(false) + .is_ok_and(|o| o.status.success()) } fn git_supports_sha256() -> bool { @@ -997,8 +1000,7 @@ fn git_supports_sha256() -> bool { dir.path().to_str().unwrap(), ]) .output() - .map(|o| o.status.success()) - .unwrap_or(false) + .is_ok_and(|o| o.status.success()) } /// A front whose `cache.max_bytes` cannot hold a repository's pack set must @@ -1815,29 +1817,42 @@ async fn partial_clone_tree_zero_and_depth_with_filter() -> TestResult { /// the instance stalled for minutes, timers included). #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn history_pack_install_does_not_stall_the_runtime() -> TestResult { - // git shim: slow only for multi-pack-index. - let shim = tempfile::tempdir()?; - let real_git = String::from_utf8( - std::process::Command::new("sh") - .args(["-c", "command -v git"]) - .output()? - .stdout, - )? - .trim() - .to_string(); - std::fs::write( - shim.path().join("git"), - format!( - "#!/bin/sh\nif [ \"$1\" = multi-pack-index ]; then sleep 3; fi\nexec {real_git} \"$@\"\n" - ), - )?; - std::fs::set_permissions( - shim.path().join("git"), - std::os::unix::fs::PermissionsExt::from_mode(0o755), - )?; - let old_path = std::env::var("PATH").unwrap_or_default(); - // SAFETY: test process, single-threaded runtime, set before any git spawn below. - unsafe { std::env::set_var("PATH", format!("{}:{old_path}", shim.path().display())) }; + const CHILD: &str = "WALGIT_TEST_HISTORY_INSTALL_CHILD"; + if std::env::var_os(CHILD).is_none() { + // git shim: slow only for multi-pack-index. + let shim = tempfile::tempdir()?; + let real_git = String::from_utf8( + std::process::Command::new("sh") + .args(["-c", "command -v git"]) + .output()? + .stdout, + )? + .trim() + .to_string(); + std::fs::write( + shim.path().join("git"), + format!( + "#!/bin/sh\nif [ \"$1\" = multi-pack-index ]; then sleep 3; fi\nexec {real_git} \"$@\"\n" + ), + )?; + std::fs::set_permissions( + shim.path().join("git"), + std::os::unix::fs::PermissionsExt::from_mode(0o755), + )?; + let old_path = std::env::var("PATH").unwrap_or_default(); + let status = tokio::process::Command::new(std::env::current_exe()?) + .args([ + "--exact", + "history_pack_install_does_not_stall_the_runtime", + "--nocapture", + ]) + .env(CHILD, "1") + .env("PATH", format!("{}:{old_path}", shim.path().display())) + .status() + .await?; + assert!(status.success(), "isolated history install test failed"); + return Ok(()); + } let big = Server::start().await?; big.put_repo("t", "hist").await?; @@ -1883,7 +1898,7 @@ async fn history_pack_install_does_not_stall_the_runtime() -> TestResult { let small = big .start_sibling_with(|c| { c.cache.prewarm = vec!["t/hist".into()]; - c.cache.prewarm_ready_timeout = std::time::Duration::from_secs(600); + c.cache.prewarm_ready_timeout = std::time::Duration::from_mins(10); }) .await?; walgit_server::prewarm::spawn(small.state.clone()); @@ -1963,18 +1978,30 @@ async fn history_pack_install_does_not_stall_the_runtime() -> TestResult { took.as_secs_f64() >= 3.0, "the shim should have slowed the install: {took:?}" ); - unsafe { std::env::set_var("PATH", old_path) }; Ok(()) } /// Materialization runs on its own runtime: even an unknown *blocking* call /// inside the install path (simulated by `WALGIT_TEST_BLOCK_INSTALL_MS`, a -/// synchronous sleep in reconcile_packs) must not stall request workers — +/// synchronous sleep in `reconcile_packs`) must not stall request workers — /// refs answer in milliseconds on a single-worker server meanwhile. #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn blocking_work_in_the_install_path_does_not_stall_requests() -> TestResult { - // SAFETY: test process; read by the sibling's sync below. - unsafe { std::env::set_var("WALGIT_TEST_BLOCK_INSTALL_MS", "2500") }; + const CHILD: &str = "WALGIT_TEST_BLOCK_INSTALL_CHILD"; + if std::env::var_os(CHILD).is_none() { + let status = tokio::process::Command::new(std::env::current_exe()?) + .args([ + "--exact", + "blocking_work_in_the_install_path_does_not_stall_requests", + "--nocapture", + ]) + .env(CHILD, "1") + .env("WALGIT_TEST_BLOCK_INSTALL_MS", "2500") + .status() + .await?; + assert!(status.success(), "isolated blocking install test failed"); + return Ok(()); + } let big = Server::start().await?; big.put_repo("t", "blk").await?; big.put_repo("t", "other2").await?; @@ -2015,7 +2042,6 @@ async fn blocking_work_in_the_install_path_does_not_stall_requests() -> TestResu worst = worst.max(t.elapsed().as_millis()); probes += 1; } - unsafe { std::env::remove_var("WALGIT_TEST_BLOCK_INSTALL_MS") }; let took = install.await?; assert!(took.as_millis() >= 2500, "{took:?}"); assert!(probes >= 5, "runtime stalled: {probes} probes in {took:?}"); diff --git a/crates/walgit-server/tests/events.rs b/crates/walgit-server/tests/events.rs index b7d8f00..dbc9898 100644 --- a/crates/walgit-server/tests/events.rs +++ b/crates/walgit-server/tests/events.rs @@ -1,8 +1,12 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::indexing_slicing, clippy::unwrap_used)] + //! Events (docs/EVENTS.md): the bridge publishes exactly what the WAL //! committed, from a durable cursor; the GCS-notification wake-up; the sweep; //! a sink failure keeps the cursor. mod harness; +use std::sync::Arc; const ZERO_OID: &str = "0000000000000000000000000000000000000000"; type TestResult = anyhow::Result<()>; @@ -14,7 +18,7 @@ type Captured = std::sync::Arc>>; /// The webhook sink's target: records every event it receives (the bus as /// the test sees it). async fn webhook() -> (String, Captured) { - let captured: Captured = Default::default(); + let captured: Captured = Arc::default(); let app = axum::Router::new().route( "/events", axum::routing::post({ diff --git a/crates/walgit-server/tests/follow.rs b/crates/walgit-server/tests/follow.rs index c7037ae..a784f0f 100644 --- a/crates/walgit-server/tests/follow.rs +++ b/crates/walgit-server/tests/follow.rs @@ -9,7 +9,7 @@ use harness::{Server, git, git_in}; macro_rules! step { ($name:literal, $e:expr) => { - tokio::time::timeout(std::time::Duration::from_secs(60), $e) + tokio::time::timeout(std::time::Duration::from_mins(1), $e) .await .unwrap_or_else(|_| panic!("step timed out: {}", $name)) }; @@ -206,8 +206,7 @@ async fn start_op( ) .await { - Ok(t) => Ok(t), - Err(walgit_server::ops::StartError::AlreadyRunning(t)) => Ok(t), + Ok(t) | Err(walgit_server::ops::StartError::AlreadyRunning(t)) => Ok(t), Err(walgit_server::ops::StartError::UnknownOp) => Err("unknown op".into()), } } diff --git a/crates/walgit-server/tests/harness.rs b/crates/walgit-server/tests/harness.rs index 85ecdb4..1e64064 100644 --- a/crates/walgit-server/tests/harness.rs +++ b/crates/walgit-server/tests/harness.rs @@ -1,3 +1,5 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::unwrap_used)] #![allow(dead_code)] //! Test harness: spin up walgit-server on a random port backed by the in-memory //! store + a tempdir cache, and drive real upstream `git` against it. @@ -76,7 +78,7 @@ impl Server { cfg.cache.max_bytes = ByteSize::gib(2); cfg.server.listen = "127.0.0.1:0".parse().unwrap(); cfg.server.max_concurrent_per_repo = 8; - cfg.server.request_timeout = std::time::Duration::from_secs(600); + cfg.server.request_timeout = std::time::Duration::from_mins(10); cfg.server.max_push_bytes = ByteSize::gib(2); cfg.wal.fsck_objects = true; cfg.wal.check_connectivity = true; @@ -96,12 +98,11 @@ impl Server { "auto" => cfg.git.upload_pack_engine = walgit_config::UploadPackEngine::Auto, _ => cfg.git.upload_pack_engine = walgit_config::UploadPackEngine::Git, } - if let Ok(ms) = std::env::var("WALGIT_TEST_MEMORY_LATENCY_MS") { - if let Ok(ms) = ms.parse::() { - if let Some(s) = Arc::get_mut(&mut store) { - s.latency = Some(std::time::Duration::from_millis(ms)); - } - } + if let Ok(ms) = std::env::var("WALGIT_TEST_MEMORY_LATENCY_MS") + && let Ok(ms) = ms.parse::() + && let Some(s) = Arc::get_mut(&mut store) + { + s.latency = Some(std::time::Duration::from_millis(ms)); } tweak(&mut cfg); @@ -144,15 +145,14 @@ impl Server { }) } - /// Two instances sharing one MemoryStore, different cache dirs. + /// Two instances sharing one `MemoryStore`, different cache dirs. pub async fn start_pair() -> Result<(Self, Self)> { let mut store = MemoryStore::shared(); - if let Ok(ms) = std::env::var("WALGIT_TEST_MEMORY_LATENCY_MS") { - if let Ok(ms) = ms.parse::() { - if let Some(s) = Arc::get_mut(&mut store) { - s.latency = Some(std::time::Duration::from_millis(ms)); - } - } + if let Ok(ms) = std::env::var("WALGIT_TEST_MEMORY_LATENCY_MS") + && let Ok(ms) = ms.parse::() + && let Some(s) = Arc::get_mut(&mut store) + { + s.latency = Some(std::time::Duration::from_millis(ms)); } let a = Self::start_with(store.clone(), tempfile::tempdir()?).await?; let b = Self::start_with(store.clone(), tempfile::tempdir()?).await?; @@ -204,7 +204,7 @@ impl Server { pub async fn registry_has_packs(&self, owner: &str, repo: &str) -> bool { let id = walgit_git::RepoId::new(owner, repo).unwrap(); match self.registry.open(&id).await { - Ok(h) => h.packs_ready() && !h.local().packs().map(|p| p.is_empty()).unwrap_or(true), + Ok(h) => h.packs_ready() && !h.local().packs().map_or(true, |p| p.is_empty()), Err(_) => false, } } @@ -216,9 +216,10 @@ impl Server { } pub async fn ls_remote(&self, owner: &str, repo: &str) -> Result { - let out = Command::new("git") + let out = tokio::process::Command::new("git") .args(["ls-remote", &self.repo_url(owner, repo)]) - .output()?; + .output() + .await?; assert!( out.status.success(), "ls-remote failed: {}", diff --git a/crates/walgit-server/tests/lfs_upstream.rs b/crates/walgit-server/tests/lfs_upstream.rs index e813235..1027673 100644 --- a/crates/walgit-server/tests/lfs_upstream.rs +++ b/crates/walgit-server/tests/lfs_upstream.rs @@ -1,3 +1,6 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::indexing_slicing, clippy::unwrap_used)] + //! `upstream.lfs` read-through (per-repo D24 setting): a mock upstream LFS //! server holds one object; walgit's store has none. //! - batch `upload`: the object is reported present with **no actions** (git-lfs @@ -9,8 +12,8 @@ mod harness; -use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use anyhow::Result; use axum::{ @@ -80,7 +83,7 @@ async fn start_mock(body: Vec) -> Result<(Arc, String)> { body, batches: AtomicUsize::new(0), downloads: AtomicUsize::new(0), - base: Default::default(), + base: Mutex::default(), }); let app = Router::new() .route("/lfs/objects/batch", post(mock_batch)) @@ -88,7 +91,7 @@ async fn start_mock(body: Vec) -> Result<(Arc, String)> { .with_state(mock.clone()); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let base = format!("http://{}", listener.local_addr()?); - *mock.base.lock().unwrap() = base.clone(); + (*mock.base.lock().unwrap()).clone_from(&base); tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); Ok((mock, base)) } diff --git a/crates/walgit-server/tests/maintain.rs b/crates/walgit-server/tests/maintain.rs index ee6c3bd..4deda1b 100644 --- a/crates/walgit-server/tests/maintain.rs +++ b/crates/walgit-server/tests/maintain.rs @@ -4,6 +4,7 @@ mod harness; use harness::{Server, git, git_in}; +use std::collections::HashMap; /// Every await is bounded so a hang names the step instead of stalling CI. macro_rules! step { @@ -16,6 +17,8 @@ macro_rules! step { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn pass_checkpoints_due_repos_refs_level_and_reports_tasks() -> anyhow::Result<()> { + use walgit_server::maintain::{Unit, next_unit, run_pass}; + // Writer front: count trigger off, so nothing auto-checkpoints on push. let front = step!("start front", Server::start())?; step!("put repo", front.put_repo("o", "r"))?; @@ -111,7 +114,6 @@ async fn pass_checkpoints_due_repos_refs_level_and_reports_tasks() -> anyhow::Re // not due), one unit per pass, next pass moves to the daily chain, and a // re-run after everything is built is idempotent (Idle). let id = walgit_git::RepoId::new("o", "r")?; - use walgit_server::maintain::{Unit, next_unit, run_pass}; assert!( matches!(step!("unit 1", next_unit(&bundler.state, &id))?, Unit::BundleSlot(ref s, _) if s == "weekly") ); @@ -277,7 +279,7 @@ async fn fsck_unit_records_missing_objects_and_repair_unit_fetches_them_from_ups c.maintenance.checkpoints = false; c.compaction.enabled = false; c.bundles.enabled = false; - c.maintenance.fsck_interval = std::time::Duration::from_secs(3600); + c.maintenance.fsck_interval = std::time::Duration::from_hours(1); }) )?; step!("put repo", server.put_repo("o", "r"))?; @@ -357,7 +359,7 @@ async fn fsck_unit_records_missing_objects_and_repair_unit_fetches_them_from_ups }; step!( "move main", - h.publish_push_synced(None, txn, Default::default()) + h.publish_push_synced(None, txn, HashMap::default()) )?; // Pass 1: the audit (never audited) → fsck.pb lists the blob; the unit succeeds (a finding, not a failure). @@ -522,7 +524,7 @@ async fn connectivity_failure_is_reported_per_ref_not_as_remote_failure() -> any }; step!( "advertise x", - h.publish_push_synced(None, txn, Default::default()) + h.publish_push_synced(None, txn, HashMap::default()) )?; // A new commit on top whose tree still references the missing blob (b.txt // unchanged): git sends commit 3 + its root tree, the server walks into b.txt. @@ -732,11 +734,7 @@ async fn bundle_list_shows_a_bundle_right_after_this_host_builds_it() -> anyhow: .await .map_err(|_| anyhow::anyhow!("op start failed"))?; assert!(t.wait_done(std::time::Duration::from_secs(30)).await); - assert!( - t.outcome().map(|o| o.is_ok()).unwrap_or(false), - "{:?}", - t.outcome() - ); + assert!(t.outcome().is_some_and(|o| o.is_ok()), "{:?}", t.outcome()); let list2 = step!("list 2", server.get_text("/o/r.git/bundles/list", &[]))?; assert!( list2.contains("[bundle \"daily-"), @@ -797,7 +795,7 @@ async fn one_pass_settles_all_closed_empty_slots() -> anyhow::Result<()> { c.maintenance.fsck_interval = std::time::Duration::ZERO; // weekly (full) + hourly on weekly: the closed hours since the weekly are empty. c.bundles.strategy.retain(|s| s.name != "daily"); - for s in c.bundles.strategy.iter_mut() { + for s in &mut c.bundles.strategy { if s.name == "hourly" { s.base = Some("weekly".into()); s.backfill_max = 0; @@ -838,7 +836,7 @@ async fn one_pass_settles_all_closed_empty_slots() -> anyhow::Result<()> { .clone(); let sunday = walgit_bundle::slots::last_slot_at_or_before( &weekly, - now - std::time::Duration::from_secs(36 * 3600), + now - std::time::Duration::from_hours(36), )? .unwrap(); let mut params = std::collections::HashMap::new(); @@ -1020,7 +1018,7 @@ async fn weekly_slot_rebuilds_the_base_then_composes_it_on_an_ssd_maintainer() - }; step!( "import refs", - h.publish_push_synced(None, txn, Default::default()) + h.publish_push_synced(None, txn, HashMap::default()) )?; step!("sync after base", h.sync())?; std::fs::write(src.path().join("g.txt"), "two\n")?; @@ -1172,12 +1170,12 @@ async fn weekly_slot_rebuilds_the_base_then_composes_it_on_an_ssd_maintainer() - "the base is the biggest tier-2 pack, not the newest" ); let next_weekly = - walgit_bundle::slots::from_epoch(weekly.slot) + std::time::Duration::from_secs(7 * 86400); + walgit_bundle::slots::from_epoch(weekly.slot) + std::time::Duration::from_hours(168); let up = walgit_server::maintain::upcoming( &h, &h.effective_config(), &walgit_server::maintain::heartbeats(&server.state).await?, - next_weekly - std::time::Duration::from_secs(60), + next_weekly - std::time::Duration::from_mins(1), ) .await; let w = up @@ -1299,7 +1297,7 @@ async fn maintainer_builds_and_publishes_missing_rev_indexes() -> anyhow::Result let task = walgit_server::ops::start(server.state.clone(), id.clone(), "rev-index", params) .await .map_err(|_| anyhow::anyhow!("rev-index op did not start"))?; - assert!(task.wait_done(std::time::Duration::from_secs(60)).await); + assert!(task.wait_done(std::time::Duration::from_mins(1)).await); assert!( matches!(task.outcome(), Some(Ok(_))), "{:?}", @@ -1366,7 +1364,7 @@ async fn identical_incremental_slots_are_skipped_as_unchanged() -> anyhow::Resul c.maintenance.checkpoints = false; c.maintenance.fsck_interval = std::time::Duration::ZERO; c.bundles.strategy.retain(|s| s.name != "daily"); - for s in c.bundles.strategy.iter_mut() { + for s in &mut c.bundles.strategy { if s.name == "hourly" { s.base = Some("weekly".into()); s.backfill_max = 0; @@ -1394,7 +1392,7 @@ async fn identical_incremental_slots_are_skipped_as_unchanged() -> anyhow::Resul let id = walgit_git::RepoId::new("o", "r")?; let h = step!("open", server.state.registry.open(&id))?; let now = std::time::SystemTime::now(); - let hour = std::time::Duration::from_secs(3600); + let hour = std::time::Duration::from_hours(1); // History with explicit times: c1 ten days ago (so a weekly slot with state // exists — a full with no state is cut from now), c2 six hours ago, nothing since. let pack_of = |revs: &str| -> anyhow::Result> { @@ -1436,7 +1434,7 @@ async fn identical_incremental_slots_are_skipped_as_unchanged() -> anyhow::Resul h.publish_push_at( Some(p1), txn("refs/heads/main", "", &c1), - Default::default(), + HashMap::default(), now - 240 * hour ) )?; @@ -1446,7 +1444,7 @@ async fn identical_incremental_slots_are_skipped_as_unchanged() -> anyhow::Resul h.publish_push_at( Some(p2), txn("refs/heads/main", &c1, &c2), - Default::default(), + HashMap::default(), now - 6 * hour ) )?; @@ -1679,7 +1677,7 @@ async fn blobless_bundle_family_is_composed_from_the_history_pack_and_served_on_ }; step!( "import refs", - h.publish_push_synced(None, txn, Default::default()) + h.publish_push_synced(None, txn, HashMap::default()) )?; std::fs::write(src.path().join("f2.txt"), "one and a half\n")?; git_in(src.path(), &["add", "."])?; @@ -1897,7 +1895,7 @@ async fn maintainer_pass_brings_an_overgrown_bundle_list_to_retention() -> anyho c.server.roles = vec![walgit_config::Role::Serve, walgit_config::Role::Maintain]; c.bundles.enabled = true; // The D21 shape this test pins (the default chains the dailies since 2026-08-22). - for s in c.bundles.strategy.iter_mut() { + for s in &mut c.bundles.strategy { s.chain = false; } }) diff --git a/crates/walgit-server/tests/routing_prefix.rs b/crates/walgit-server/tests/routing_prefix.rs index 5c65c55..4205666 100644 --- a/crates/walgit-server/tests/routing_prefix.rs +++ b/crates/walgit-server/tests/routing_prefix.rs @@ -1,3 +1,10 @@ +#![allow( + clippy::case_sensitive_file_extension_comparisons, + clippy::unnecessary_wraps +)] +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::panic, clippy::string_slice)] + //! D26/D27 + no-compat banner: **repo prefix first, lane segment second**. //! Source-level (grep), not HTTP. //! @@ -108,7 +115,7 @@ fn forbidden_client_hits(src: &str, rel: &str) -> Vec { let mut hits = Vec::new(); for (i, line) in src.lines().enumerate() { let t = line.trim(); - if t.starts_with("//") || t.starts_with("*") || t.starts_with("/*") { + if t.starts_with("//") || t.starts_with('*') || t.starts_with("/*") { continue; } // Documentation of the alias in comments is fine; code that builds a URL is not. @@ -159,8 +166,6 @@ fn clients_emit_prefix_form() -> TestResult { } fn walk_ts(dir: &str) -> Vec<(String, String)> { - let mut out = Vec::new(); - let base = root().join(dir); fn rec(dir: &Path, root: &Path, out: &mut Vec<(String, String)>) { let Ok(rd) = fs::read_dir(dir) else { return }; for e in rd.flatten() { @@ -181,6 +186,10 @@ fn walk_ts(dir: &str) -> Vec<(String, String)> { } } } + + let mut out = Vec::new(); + let base = root().join(dir); + rec(&base, &root(), &mut out); out } diff --git a/crates/walgit-server/tests/sim.rs b/crates/walgit-server/tests/sim.rs index 65e2026..1d6cf93 100644 --- a/crates/walgit-server/tests/sim.rs +++ b/crates/walgit-server/tests/sim.rs @@ -1,4 +1,7 @@ -//! Simulation tests: safety mode → liveness mode (after TigerBeetle's VOPR, +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::expect_used, clippy::indexing_slicing, clippy::unwrap_used)] + +//! Simulation tests: safety mode → liveness mode (after `TigerBeetle`'s VOPR, //! "Simulation Testing For Liveness", 2023). //! //! A *cluster* is N walgit instances (one `Registry` + cache dir each) that @@ -111,7 +114,9 @@ impl WorkRepo { fn pack(&self, head: &str, base: Option<&str>) -> Vec { let mut revs = format!("{head}\n"); if let Some(b) = base { - revs.push_str(&format!("^{b}\n")); + { + let _ = std::fmt::Write::write_fmt(&mut revs, format_args!("^{b}\n")); + }; } let mut child = Command::new("git") .args(["pack-objects", "--stdout", "--revs", "-q"]) @@ -167,7 +172,7 @@ struct Instance { link: Arc, registry: Arc, cfg: Arc, - _cache: tempfile::TempDir, + cache: tempfile::TempDir, } impl Instance { @@ -198,7 +203,7 @@ impl Instance { link, registry, cfg, - _cache: cache, + cache, } } async fn open(&self, id: &RepoId) -> Result> { @@ -218,7 +223,7 @@ struct Cluster { impl Cluster { async fn new(seed: u64, n: usize) -> Result { let truth: DynStore = MemoryStore::shared(); - let id = RepoId::new("sim", &format!("r{seed}"))?; + let id = RepoId::new("sim", format!("r{seed}"))?; let mut c = Cluster { seed, truth, @@ -258,7 +263,7 @@ impl Cluster { let s = self.next_link_seed.fetch_add(1, Ordering::Relaxed); // Take the cache dir out of the old instance without dropping it. let placeholder = tempfile::tempdir().unwrap(); - let cache = std::mem::replace(&mut self.instances[i]._cache, placeholder); + let cache = std::mem::replace(&mut self.instances[i].cache, placeholder); let fresh = Instance::new_at(&self.truth, &name, s, cache, tweak); let old = std::mem::replace(&mut self.instances[i], fresh); drop(old); @@ -282,11 +287,12 @@ impl Cluster { fn dump_traces(&self) -> String { let mut s = String::new(); for i in &self.instances { - s.push_str(&format!( - "--- link {} ({})\n", - i.name, - i.link.stats().summary() - )); + { + let _ = std::fmt::Write::write_fmt( + &mut s, + format_args!("--- link {} ({})\n", i.name, i.link.stats().summary()), + ); + }; for l in i .link .take_trace() @@ -305,7 +311,7 @@ impl Cluster { } } -/// BundleSource adapter used by the bundle-lease liveness scenario. +/// `BundleSource` adapter used by the bundle-lease liveness scenario. struct SimBundleSource(Arc); #[async_trait::async_trait] @@ -522,7 +528,7 @@ async fn check_truth(c: &Cluster, pushers: &[Pusher]) -> Result<()> { // The checkpoint (if any) folds the log prefix: refs from its RefSnapshot, // entries after it from the tail. Both must exist in the bucket. let prefix = c.repo_prefix(); - let cp_seq = manifest.checkpoint.as_ref().map(|cp| cp.seq).unwrap_or(0); + let cp_seq = manifest.checkpoint.as_ref().map_or(0, |cp| cp.seq); let mut folded: HashMap = HashMap::new(); if cp_seq > 0 { let key = format!( @@ -559,14 +565,14 @@ async fn check_truth(c: &Cluster, pushers: &[Pusher]) -> Result<()> { ); } ensure!( - log.first().map(|e| e.seq > cp_seq).unwrap_or(true), + log.first().is_none_or(|e| e.seq > cp_seq), "log tail starts at {} <= checkpoint {cp_seq}", log[0].seq ); ensure!( - log.last().map(|e| e.seq).unwrap_or(cp_seq) == manifest.head_seq, + log.last().map_or(cp_seq, |e| e.seq) == manifest.head_seq, "log tail {} != manifest.head_seq {}", - log.last().map(|e| e.seq).unwrap_or(cp_seq), + log.last().map_or(cp_seq, |e| e.seq), manifest.head_seq ); // Every ACK after the checkpoint is in the log at its seq with its txn. @@ -615,14 +621,11 @@ async fn check_truth(c: &Cluster, pushers: &[Pusher]) -> Result<()> { // f must be last.new or a commit pushed after it (the ack'd or an // errored-but-committed push along the same chain). let later = log.iter().filter(|e| e.seq > last.seq).any(|e| { - e.txn - .as_ref() - .map(|t| { - t.updates - .iter() - .any(|u| u.name == p.refname && u.new_oid == f) - }) - .unwrap_or(false) + e.txn.as_ref().is_some_and(|t| { + t.updates + .iter() + .any(|u| u.name == p.refname && u.new_oid == f) + }) }) || last.seq <= cp_seq; ensure!( f == last.new || later, @@ -787,8 +790,10 @@ async fn check_core_liveness( .await .map_err(|_| anyhow!("liveness: compaction hung > {bound:?}"))?; match out { - Ok(walgit_server::ops::CompactOutcome::Published { .. }) - | Ok(walgit_server::ops::CompactOutcome::NotTriggered { .. }) => break, + Ok( + walgit_server::ops::CompactOutcome::Published { .. } + | walgit_server::ops::CompactOutcome::NotTriggered { .. }, + ) => break, Ok(walgit_server::ops::CompactOutcome::LeaseHeld) => { ensure!( t.elapsed() < bound, @@ -834,7 +839,7 @@ fn seeds() -> Vec { .ok() .and_then(|s| s.parse().ok()) .unwrap_or(2); - (1..=n).map(|i| 0xC0FFEE + i * 7919).collect() + (1..=n).map(|i| 0x00C0_FFEE + i * 7_919).collect() } fn pushes_per_pusher() -> u64 { std::env::var("WALGIT_SIM_PUSHES") @@ -848,15 +853,20 @@ impl Lcg { fn next(&mut self) -> u64 { self.0 = self .0 - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); self.0 >> 33 } fn below(&mut self, n: u64) -> u64 { self.next() % n.max(1) } + fn below_usize(&mut self, n: usize) -> usize { + let bound = u64::try_from(n).expect("usize always fits in u64"); + usize::try_from(self.below(bound)).expect("random value is less than the usize bound") + } fn chance(&mut self, p: f64) -> bool { - (self.next() as f64 / (1u64 << 31) as f64) < p + let sample = u32::try_from(self.next()).expect("LCG output is limited to 31 bits"); + (f64::from(sample) / f64::from(1u32 << 31)) < p } } @@ -876,19 +886,19 @@ async fn run_safety_then_liveness(seed: u64) -> Result<()> { let per = pushes_per_pusher(); let op_timeout = Duration::from_secs(10); for round in 0..per { - for p in pushers.iter_mut() { - let i = rng.below(n_instances as u64) as usize; + for p in &mut pushers { + let i = rng.below_usize(n_instances); let _ = p.push_once(&c.instances[i], &c.id, op_timeout).await?; } // Random crash: replace an instance (its in-flight state is gone). if rng.chance(0.2) { - let i = rng.below(n_instances as u64) as usize; + let i = rng.below_usize(n_instances); c.restart(i); c.instances[i].link.set(FaultPlan::chaos(0.04)); } // Occasionally somebody checkpoints or compacts under chaos. if round % 4 == 3 { - let i = rng.below(n_instances as u64) as usize; + let i = rng.below_usize(n_instances); if let Ok(h) = c.instances[i].open(&c.id).await { let _ = tokio::time::timeout(op_timeout, h.write_checkpoint()).await; let cfg = c.instances[i].cfg.clone(); @@ -922,7 +932,7 @@ async fn run_safety_then_liveness(seed: u64) -> Result<()> { // Liveness mode: pick a core of 2, heal it, freeze the rest in nasty states. let mut idx: Vec = (0..n_instances).collect(); for k in (1..idx.len()).rev() { - let j = rng.below(k as u64 + 1) as usize; + let j = rng.below_usize(k + 1); idx.swap(k, j); } let core = &idx[..2]; @@ -951,7 +961,7 @@ async fn run_safety_then_liveness(seed: u64) -> Result<()> { for (k, &i) in idx[2..].iter().enumerate() { c.instances[i] .link - .set(frozen[(k + rng.below(4) as usize) % frozen.len()].clone()); + .set(frozen[(k + rng.below_usize(4)) % frozen.len()].clone()); } // Non-core pushers keep hammering the frozen links in the background (they // may never interfere with the core). @@ -968,7 +978,7 @@ async fn run_safety_then_liveness(seed: u64) -> Result<()> { link, registry: reg, cfg: Arc::new(sim_config(Path::new("/nonexistent"))), - _cache: tempfile::tempdir().unwrap(), + cache: tempfile::tempdir().unwrap(), }; for _ in 0..20 { let _ = p.push_once(&inst, &id, Duration::from_millis(500)).await; @@ -1005,7 +1015,7 @@ async fn sim_safety_then_liveness() { let r = run_safety_then_liveness(seed).await; eprintln!( "[seed {seed}] {:?} in {:.1}s", - r.as_ref().map(|_| "ok"), + r.as_ref().map(|()| "ok"), t.elapsed().as_secs_f64() ); if let Err(e) = r { @@ -1066,7 +1076,9 @@ async fn liveness_compaction_after_lease_holder_dies() -> Result<()> { ); tokio::time::sleep(Duration::from_millis(200)).await; } - other => bail!("unexpected {other:?}"), + other @ walgit_server::ops::CompactOutcome::NotTriggered { .. } => { + bail!("unexpected {other:?}") + } } } eprintln!( @@ -1151,13 +1163,13 @@ async fn liveness_stale_instance_cannot_starve_the_core() -> Result<()> { link: stale_link, registry: stale_reg, cfg: Arc::new(sim_config(Path::new("/nonexistent"))), - _cache: tempfile::tempdir().unwrap(), + cache: tempfile::tempdir().unwrap(), }; let mut n = 0u64; loop { let _ = stale_p.push_once(&inst, &id, Duration::from_secs(2)).await; n += 1; - if n % 10 == 0 { + if n.is_multiple_of(10) { tracing::info!( "stale pusher: {n} attempts, last: {:?}", stale_p.errors.last() @@ -1338,6 +1350,7 @@ async fn liveness_orphaned_log_segment_does_not_block_writers() -> Result<()> { /// Once its link heals, it must finish syncing — a half-downloaded pack on /// disk may not poison every later attempt. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[allow(clippy::many_single_char_names)] async fn liveness_cold_start_through_truncated_pack_reads() -> Result<()> { let mut c = Cluster::new(15, 1).await?; let mut p = Pusher::new(0); @@ -1436,7 +1449,7 @@ async fn liveness_black_holed_instance_is_invisible_to_the_core() -> Result<()> link, registry: reg, cfg: Arc::new(sim_config(Path::new("/nonexistent"))), - _cache: tempfile::tempdir().unwrap(), + cache: tempfile::tempdir().unwrap(), }; for _ in 0..5 { let _ = p1.push_once(&inst, &id, Duration::from_secs(30)).await; @@ -1519,7 +1532,7 @@ async fn liveness_frozen_task_owner_does_not_wedge_readiness() -> Result<()> { Ok(()) } -/// A request ReadGuard is the pin that promises packs remain on disk. Even a +/// A request `ReadGuard` is the pin that promises packs remain on disk. Even a /// leaked guard must make eviction skip the repo; after it drops, eviction may /// reclaim the cache. #[tokio::test] @@ -1638,7 +1651,7 @@ async fn liveness_bundle_build_after_lease_holder_dies() -> Result<()> { } /// Exact healthy-link request counts defend the critical-path budgets in -/// docs/ROUNDTRIPS.md. MemoryStore has no retries, so deltas are deterministic: +/// docs/ROUNDTRIPS.md. `MemoryStore` has no retries, so deltas are deterministic: /// push = one freshness GET + pack/idx/log PUTs + manifest CAS; warm refs = one /// conditional GET; cold refs = the open's manifest GET + one log tail GET. #[tokio::test] @@ -2027,6 +2040,7 @@ fn pack_objects(repo: &Path, checksum: &gix_hash::ObjectId) -> std::collections: /// Build a large-repository shape on a disk-mode host: a tier-2 base (full repack + bitmap) with its D18 /// history pack, then several fresh pushes. Returns (base, history) checksums. +#[allow(clippy::many_single_char_names)] async fn seed_base_and_history( c: &Cluster, i: usize, @@ -2194,7 +2208,7 @@ async fn full_rebuild_leaves_exactly_one_base_even_with_a_retained_pack() -> Res cfg.git.history_pack = true; }); let mut p = Pusher::new(0); - let (base1, _hist1) = seed_base_and_history(&c, i, &mut p, 2).await?; + let (_base1, _hist1) = seed_base_and_history(&c, i, &mut p, 2).await?; let h = c.instances[i].open(&c.id).await?; drop(h.sync_full().await?); // Simulate git retaining the old base (a `.keep` git would honour — as a kept pack it is not @@ -2265,7 +2279,6 @@ async fn full_rebuild_leaves_exactly_one_base_even_with_a_retained_pack() -> Res "rebuild superseded {superseded} of {} live packs", before.len() ); - ensure!(base1 != gix_hash::ObjectId::from_hex(fulls[0].checksum.as_bytes())? || true); check_truth(&c, std::slice::from_ref(&p)).await?; Ok(()) } @@ -2282,7 +2295,7 @@ async fn rebuild_attempt( let lines: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); let h = match c.instances[i].open(&c.id).await { Ok(h) => h, - Err(e) => return (Err(e.into()), Vec::new()), + Err(e) => return (Err(e), Vec::new()), }; let out = walgit_server::ops::compact_repo( &h, @@ -2303,6 +2316,7 @@ async fn rebuild_attempt( /// result is one base + one history pack. A push between the attempts makes the head move, and /// the next unit starts over (a second repack) instead of publishing a pack that lacks objects. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[allow(clippy::many_single_char_names)] async fn base_rebuild_resumes_after_a_kill_between_any_two_phases() -> Result<()> { use walgit_server::rebuild::{Phase, TEST_ABORT_AFTER}; let mut c = Cluster::new(33, 1).await?; @@ -2471,8 +2485,8 @@ async fn base_rebuild_resumes_after_a_kill_between_any_two_phases() -> Result<() /// download and caches have something to evict). fn push_blobby(p: &mut Pusher, kb: usize, rng: &mut Lcg) -> String { let mut buf = vec![0u8; kb * 1024]; - for b in buf.iter_mut() { - *b = rng.next() as u8; + for b in &mut buf { + *b = u8::try_from(rng.next() & 0xff).expect("masked random byte fits in u8"); } std::fs::write(p.work.path().join(format!("blob-{}.bin", p.n + 1)), &buf).unwrap(); p.work.commit(p.n + 1, &format!("p{}", p.idx)) @@ -2485,6 +2499,7 @@ fn push_blobby(p: &mut Pusher, kb: usize, rng: &mut Lcg) -> String { /// `materialize` running for the repo; an aborted owner releases the lock at once (the next /// caller starts its own task — nothing blocks forever); a late joiner's `attach()` replays /// the story so far and sees the outcome; downloads are not multiplied by the callers. +#[allow(clippy::many_single_char_names)] async fn run_task_ownership(seed: u64) -> Result<()> { let mut rng = Lcg(seed); let mut c = Cluster::new(seed, 1).await?; @@ -2530,7 +2545,8 @@ async fn run_task_ownership(seed: u64) -> Result<()> { Duration::from_millis(1), Duration::from_millis(2 + rng.below(15)), )), - p_err_before: 0.05 + (rng.below(10) as f64) / 100.0, + p_err_before: 0.05 + + f64::from(u32::try_from(rng.below(10)).expect("sample is below 10")) / 100.0, p_truncate: 0.05, ..Default::default() } @@ -2541,15 +2557,15 @@ async fn run_task_ownership(seed: u64) -> Result<()> { let repo = c.id.to_string(); // K concurrent object-level syncs; one random caller is aborted after a random delay. - let k = 4 + rng.below(4) as usize; + let k = 4 + rng.below_usize(4); let mut joins = Vec::new(); for _ in 0..k { let h = h.clone(); joins.push(tokio::spawn(async move { - h.sync().await.map(|g| drop(g)).map_err(|e| e.to_string()) + h.sync().await.map(drop).map_err(|e| e.to_string()) })); } - let victim = rng.below(k as u64) as usize; + let victim = rng.below_usize(k); let abort_after = Duration::from_millis(rng.below(40)); // Watch the task registry while they run: at most one materialize task at a time. let watcher = { @@ -2629,7 +2645,8 @@ async fn run_task_ownership(seed: u64) -> Result<()> { "late joiner did not see the outcome: {outcome:?}" ); // Downloads: every attempt downloads each pack at most once (+ idx); no N-fold traffic. - let ops = c.instances[j].link.stats().ops.load(Ordering::Relaxed) as usize; + let ops = usize::try_from(c.instances[j].link.stats().ops.load(Ordering::Relaxed)) + .context("store operation count does not fit usize")?; let attempts = materializes.len(); let budget = attempts * (live_packs * 4 + 6) + k * 3 + 20; ensure!( @@ -2651,10 +2668,10 @@ async fn sim_task_ownership_under_concurrency_and_owner_crash() { /// Budget-mode cache pressure: four repositories of which the cache holds about two, a /// randomized interleaving of refs-level and object-level reads, one repository pinned by a -/// live ReadGuard throughout, plus one repository whose pack set exceeds `cache.max_bytes`. +/// live `ReadGuard` throughout, plus one repository whose pack set exceeds `cache.max_bytes`. /// Asserted after every step: the pinned repo is never evicted; the too-large repo is refused /// with `TooLarge` (never materialized, never the cause of evicting the others); the cache -/// stays ≤ max_bytes + one pack set; a refs-level read on a cold repo during eviction stays fast. +/// stays ≤ `max_bytes` + one pack set; a refs-level read on a cold repo during eviction stays fast. async fn run_cache_pressure(seed: u64) -> Result<()> { let mut rng = Lcg(seed ^ 0x5EED); let truth: DynStore = MemoryStore::shared(); @@ -2662,7 +2679,7 @@ async fn run_cache_pressure(seed: u64) -> Result<()> { let mut ids = Vec::new(); let mut pushers = Vec::new(); for r in 0..4u32 { - let id = RepoId::new("sim", &format!("cache{seed}-{r}"))?; + let id = RepoId::new("sim", format!("cache{seed}-{r}"))?; writer.registry.create(&id, ObjectFormat::Sha1).await?; let mut p = Pusher::new(r as usize); let new = push_blobby(&mut p, 96, &mut rng); @@ -2694,7 +2711,7 @@ async fn run_cache_pressure(seed: u64) -> Result<()> { pushers.push(p); } // The big one: ~5 × a small repo. - let big = RepoId::new("sim", &format!("cache{seed}-big"))?; + let big = RepoId::new("sim", format!("cache{seed}-big"))?; writer.registry.create(&big, ObjectFormat::Sha1).await?; { let mut p = Pusher::new(9); @@ -2752,7 +2769,7 @@ async fn run_cache_pressure(seed: u64) -> Result<()> { let mut refs_latencies = Vec::new(); let mut total_evicted = 0usize; for step in 0..30u64 { - let r = 1 + rng.below(3) as usize; // repos 1..3 + let r = 1 + rng.below_usize(3); // repos 1..3 let id = &ids[r]; let h = front.registry.open(id).await?; match rng.below(3) { diff --git a/crates/walgit-server/tests/static_http.rs b/crates/walgit-server/tests/static_http.rs index ff64fdf..ed7f015 100644 --- a/crates/walgit-server/tests/static_http.rs +++ b/crates/walgit-server/tests/static_http.rs @@ -1,5 +1,5 @@ //! HTTP contract of immutable store objects (LFS here; bundles share the same -//! `static_object` path) and of the embedded UI assets: strong ETags, 304, +//! `static_object` path) and of the embedded UI assets: strong `ETags`, 304, //! Range/If-Range, HEAD, Content-Length, precompressed encodings. mod harness; @@ -206,7 +206,12 @@ async fn ui_assets_etag_304_and_precompressed() -> Result<()> { // same ETag across encodings (the encoding is negotiated, not a new entity). let asset = html .split('"') - .find(|p| p.starts_with("/_ui/assets/") && p.ends_with(".js")) + .find(|p| { + p.starts_with("/_ui/assets/") + && std::path::Path::new(p) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("js")) + }) .expect("asset reference") .to_string(); let url = format!("{}{}", server.base_url, asset); diff --git a/crates/walgit-server/tests/web_api.rs b/crates/walgit-server/tests/web_api.rs index 4b55379..f90cec1 100644 --- a/crates/walgit-server/tests/web_api.rs +++ b/crates/walgit-server/tests/web_api.rs @@ -1,3 +1,6 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::indexing_slicing, clippy::string_slice, clippy::unwrap_used)] + //! web/API.md §6 conformance for the read-only JSON API. mod harness; @@ -21,7 +24,7 @@ async fn get( .headers() .get("content-type") .and_then(|v| v.to_str().ok()) - .map(|s| s.to_string()); + .map(std::string::ToString::to_string); let text = resp.text().await?; Ok((status, text, ct)) } @@ -112,6 +115,7 @@ fn fixture(server: &Server) -> anyhow::Result { /// web/API.md §6 against one server (called for the local-packs instance and /// for a sibling that serves the same repo remotely). +#[allow(clippy::many_single_char_names)] async fn conformance( server: &Server, src: &std::path::Path, diff --git a/crates/walgit-server/tests/web_ui.rs b/crates/walgit-server/tests/web_ui.rs index 276484c..81cb64d 100644 --- a/crates/walgit-server/tests/web_ui.rs +++ b/crates/walgit-server/tests/web_ui.rs @@ -55,7 +55,12 @@ async fn assets_have_content_type_and_immutable_cache() -> Result<()> { let marker = "/_ui/assets/"; let asset = index .split('"') - .find(|part| part.starts_with(marker) && part.ends_with(".js")) + .find(|part| { + part.starts_with(marker) + && std::path::Path::new(part) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("js")) + }) .expect("built index references a JavaScript asset"); let response = client .get(format!("{}{}", server.base_url, asset)) diff --git a/crates/walgit-store/src/coord.rs b/crates/walgit-store/src/coord.rs index 365b8e9..3e3317e 100644 --- a/crates/walgit-store/src/coord.rs +++ b/crates/walgit-store/src/coord.rs @@ -133,9 +133,8 @@ where T: prost::Message + Default, { match store.get_if_changed(key, known).await { - Err(StoreError::NotFound { .. }) => Ok(None), + Err(StoreError::NotFound { .. }) | Ok(None) => Ok(None), Err(e) => Err(CoordError::Store(e)), - Ok(None) => Ok(None), Ok(Some((meta, bytes))) => { let msg = T::decode(bytes)?; Ok(Some((meta, msg))) @@ -173,6 +172,10 @@ pub struct LeaseGuard { } impl LeaseGuard { + #[expect( + clippy::too_many_arguments, + reason = "Lease construction collects its store identity and timing in one place" + )] fn new( store: DynStore, key: &str, @@ -226,9 +229,9 @@ impl LeaseGuard { .delete(&self.key, Some(self.version.clone())) .await { - Ok(()) - | Err(StoreError::PreconditionFailed { .. }) - | Err(StoreError::NotFound { .. }) => Ok(()), + Ok(()) | Err(StoreError::PreconditionFailed { .. } | StoreError::NotFound { .. }) => { + Ok(()) + } Err(e) => Err(CoordError::Store(e)), } } @@ -286,9 +289,9 @@ impl Drop for LeaseGuard { let key = self.key.clone(); let version = self.version.clone(); if let Ok(handle) = tokio::runtime::Handle::try_current() { - let _ = handle.spawn(async move { + drop(handle.spawn(async move { let _ = store.delete(&key, Some(version)).await; - }); + })); } } } @@ -327,8 +330,7 @@ pub async fn try_acquire( let expires_at = existing .expires_at .as_ref() - .map(time::to_system) - .unwrap_or(UNIX_EPOCH); + .map_or(UNIX_EPOCH, time::to_system); if now >= expires_at + LEASE_SKEW_TOLERANCE { let epoch = existing.epoch + 1; let lease = make_lease(holder, purpose, now, ttl, epoch); @@ -422,9 +424,10 @@ mod tests { #[tokio::test] async fn cas_update_convergence_64_incrementers() { + const N: u32 = 64; + let store = dyn_store(); let key = "counter.pb"; - const N: u32 = 64; let mut handles = Vec::new(); for i in 0..N { @@ -465,9 +468,10 @@ mod tests { #[tokio::test] async fn lease_exclusivity_32_concurrent() { + const N: u32 = 32; + let store = dyn_store(); let key = "leases/excl.pb"; - const N: u32 = 32; let mut handles = Vec::new(); for i in 0..N { @@ -475,7 +479,7 @@ mod tests { let k = key.to_string(); handles.push(tokio::spawn(async move { let holder = format!("h{i}"); - try_acquire(s, &k, &holder, "test", Duration::from_secs(60)).await + try_acquire(s, &k, &holder, "test", Duration::from_mins(1)).await })); } let mut successes = 0; @@ -614,7 +618,7 @@ mod tests { let store = dyn_store(); let key = "leases/timeout.pb"; - let _g1 = try_acquire(store.clone(), key, "h1", "test", Duration::from_secs(60)) + let _g1 = try_acquire(store.clone(), key, "h1", "test", Duration::from_mins(1)) .await .unwrap() .unwrap(); diff --git a/crates/walgit-store/src/fault.rs b/crates/walgit-store/src/fault.rs index ac1c988..47df311 100644 --- a/crates/walgit-store/src/fault.rs +++ b/crates/walgit-store/src/fault.rs @@ -108,8 +108,9 @@ impl FaultPlan { ..Default::default() } } + #[must_use] pub fn with_only(mut self, keys: &[&str]) -> Self { - self.only_keys = Some(keys.iter().map(|s| s.to_string()).collect()); + self.only_keys = Some(keys.iter().map(std::string::ToString::to_string).collect()); self } } @@ -168,6 +169,10 @@ impl Rng { self.0 = x; x.wrapping_mul(0x2545_F491_4F6C_DD1D) } + #[expect( + clippy::cast_precision_loss, + reason = "The shifted numerator has 53 bits and the denominator is exactly 2^53" + )] fn f64(&mut self) -> f64 { (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64 } @@ -257,6 +262,14 @@ impl FaultStore { /// Roll the dice for one op. `mutation`: put/delete/compose; `conditional`: /// CAS put/delete or if-none-match get; `body_len`: for truncation. + #[expect( + clippy::cast_possible_truncation, + reason = "The truncation offset is sampled below 2^20 and fits usize" + )] + #[expect( + clippy::panic, + reason = "Explicit crash injection is the purpose of the fault-store test adapter" + )] async fn decide( &self, op: &str, @@ -300,8 +313,8 @@ impl FaultStore { return Decision::Denied; } if let Some((lo, hi)) = plan.delay { - let span = hi.saturating_sub(lo).as_micros() as u64; - let extra = self.rng.lock().below(span + 1); + let span = u64::try_from(hi.saturating_sub(lo).as_micros()).unwrap_or(u64::MAX); + let extra = self.rng.lock().below(span.saturating_add(1)); tokio::time::sleep(lo + Duration::from_micros(extra)).await; } if !Self::in_scope(&plan, key) { @@ -543,6 +556,41 @@ pub async fn truth_bytes(store: &DynStore, key: &str) -> Result> { Ok(store.get_bytes(key).await?.map(|(_, b)| b)) } +impl FaultStore { + async fn get_inner(&self, key: &str, opts: GetOptions, conditional: bool) -> Result { + match self.decide("get", key, false, conditional, true).await { + Decision::Hang => hang_forever().await, + Decision::ErrBefore => Err(self.retryable("get", key, "before")), + Decision::Denied => Err(StoreError::NotFound { key: key.into() }), + Decision::Stale => Ok(GetResult::NotModified { + version: opts.if_none_match.clone().ok_or_else(|| { + StoreError::other(anyhow::anyhow!( + "stale response needs a conditional version" + )) + })?, + }), + Decision::Truncate(at) => match self.inner.get(key, opts).await? { + GetResult::Object { meta, body } => { + let size = usize::try_from(meta.size).map_err(StoreError::other)?; + let at = if size == 0 { 0 } else { at % size }; + let msg = format!( + "fault-store[{}]: injected truncation of {key} at {at}/{size}", + self.name + ); + Ok(GetResult::Object { + meta, + body: truncate_stream(body, at, msg), + }) + } + r @ GetResult::NotModified { .. } => Ok(r), + }, + Decision::Proceed | Decision::ErrAfter | Decision::CasFail => { + self.inner.get(key, opts).await + } + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -611,34 +659,3 @@ mod tests { assert!(r.is_err()); } } - -impl FaultStore { - async fn get_inner(&self, key: &str, opts: GetOptions, conditional: bool) -> Result { - match self.decide("get", key, false, conditional, true).await { - Decision::Hang => hang_forever().await, - Decision::ErrBefore => Err(self.retryable("get", key, "before")), - Decision::Denied => Err(StoreError::NotFound { key: key.into() }), - Decision::Stale => Ok(GetResult::NotModified { - version: opts.if_none_match.clone().unwrap(), - }), - Decision::Truncate(at) => match self.inner.get(key, opts).await? { - GetResult::Object { meta, body } => { - let size = meta.size as usize; - let at = if size == 0 { 0 } else { at % size }; - let msg = format!( - "fault-store[{}]: injected truncation of {key} at {at}/{size}", - self.name - ); - Ok(GetResult::Object { - meta, - body: truncate_stream(body, at, msg), - }) - } - r => Ok(r), - }, - Decision::Proceed | Decision::ErrAfter | Decision::CasFail => { - self.inner.get(key, opts).await - } - } - } -} diff --git a/crates/walgit-store/src/gcs.rs b/crates/walgit-store/src/gcs.rs index 071b63c..8e7d68c 100644 --- a/crates/walgit-store/src/gcs.rs +++ b/crates/walgit-store/src/gcs.rs @@ -39,9 +39,9 @@ const LIST_PAGE_SIZE: i32 = 1000; /// Mid-stream resumes per bulk read before the error is surfaced. const BULK_RESUME_ATTEMPTS: u32 = 5; const META_DEADLINE: std::time::Duration = std::time::Duration::from_secs(10); -const READ_OPEN_DEADLINE: std::time::Duration = std::time::Duration::from_secs(60); +const READ_OPEN_DEADLINE: std::time::Duration = std::time::Duration::from_mins(1); /// Per chunk of a streaming body read (not the whole stream). -const READ_CHUNK_DEADLINE: std::time::Duration = std::time::Duration::from_secs(60); +const READ_CHUNK_DEADLINE: std::time::Duration = std::time::Duration::from_mins(1); const PUT_MIN_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30); /// Uploads get this many bytes per second on top of `PUT_MIN_DEADLINE` (1 MiB/s floor). const PUT_BYTES_PER_SEC: u64 = 1024 * 1024; @@ -56,7 +56,7 @@ fn deadline_error(op: &str, key: &str, deadline: std::time::Duration) -> StoreEr tracing::warn!( op, key, - deadline_ms = deadline.as_millis() as u64, + deadline_ms = u64::try_from(deadline.as_millis()).unwrap_or(u64::MAX), "gcs call exceeded deadline" ); StoreError::retryable(anyhow::anyhow!( @@ -65,7 +65,7 @@ fn deadline_error(op: &str, key: &str, deadline: std::time::Duration) -> StoreEr } /// Run a GCS call under `deadline`; the client error keeps its meaning through -/// `map_error` (NotFound / PreconditionFailed / NotModified), a timeout becomes +/// `map_error` (`NotFound` / `PreconditionFailed` / `NotModified`), a timeout becomes /// [`deadline_error`]. `retries` extra attempts are made only when the deadline /// fired (the call is idempotent for every caller that passes > 0). async fn call( @@ -92,16 +92,17 @@ where op, key, attempt, - deadline_ms = deadline.as_millis() as u64, + deadline_ms = u64::try_from(deadline.as_millis()).unwrap_or(u64::MAX), "gcs call exceeded deadline, retrying" ); attempt += 1; let jitter = 100 - + (std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.subsec_nanos()) - .unwrap_or(0) - % 400) as u64; + + u64::from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.subsec_nanos()) + % 400, + ); tokio::time::sleep(std::time::Duration::from_millis(jitter)).await; } } @@ -169,6 +170,7 @@ impl GcsStore { /// data (`Storage`) and control (`StorageControl`) clients, allowing /// emulator use. /// Set `telemetry.lock_wait_warn` for the bulk-permit WARN line (default 1 s). + #[must_use] pub fn with_permit_wait_warn(mut self, d: std::time::Duration) -> Self { self.permit_wait_warn = d; self @@ -237,6 +239,10 @@ impl GcsStore { /// Bulk keys: pack data and side-files, bundles, LFS (everything that is /// large or read by range); the rest is control plane. + #[expect( + clippy::case_sensitive_file_extension_comparisons, + reason = "Object store control keys use exact case-sensitive suffixes" + )] fn is_bulk_key(key: &str) -> bool { // Pack data + side-files, bundle *files* (not `bundles/list.pb`), LFS // objects. Everything else — manifest, log, checkpoints, leases, @@ -270,6 +276,14 @@ impl GcsStore { } /// The data client for `key` (+ a bulk permit when it is bulk traffic). + #[expect( + clippy::cast_precision_loss, + reason = "In-flight permit count is an approximate metric" + )] + #[expect( + clippy::indexing_slicing, + reason = "Construction guarantees a nonempty client pool; the index is modulo its length" + )] async fn data_client( &self, key: &str, @@ -288,7 +302,10 @@ impl GcsStore { let permit = self.bulk_permits.clone().acquire_owned().await.ok(); let queued = t.elapsed(); if queued.as_millis() > 0 { - tracing::Span::current().record("queued_ms", queued.as_millis() as u64); + tracing::Span::current().record( + "queued_ms", + u64::try_from(queued.as_millis()).unwrap_or(u64::MAX), + ); } metrics::histogram!("walgit_store_bulk_queue_seconds").record(queued.as_secs_f64()); if queued > std::time::Duration::ZERO { @@ -299,7 +316,7 @@ impl GcsStore { tracing::warn!( lock = "gcs_bulk_permit", key, - wait_ms = queued.as_millis() as u64, + wait_ms = u64::try_from(queued.as_millis()).unwrap_or(u64::MAX), "lock wait" ); } @@ -314,7 +331,7 @@ impl GcsStore { fn meta_from_object(obj: &google_cloud_storage::model::Object) -> ObjectMeta { ObjectMeta { key: obj.name.clone(), - size: obj.size as u64, + size: obj.size.max(0).cast_unsigned(), version: gen_version(obj.generation), } } @@ -364,16 +381,18 @@ impl BulkHttp { range: Option>, if_generation_match: Option, ) -> Result<(u64, Option, ByteStream)> { - let (size, generation, first) = self.open(key, range.clone(), if_generation_match).await?; - let end = range.as_ref().map(|r| r.end).unwrap_or(size); - let start = range.as_ref().map(|r| r.start).unwrap_or(0); - let this = self.clone(); - let key_owned = key.to_owned(); struct St { inner: ByteStream, pos: u64, attempts: u32, } + + let (size, generation, first) = self.open(key, range.clone(), if_generation_match).await?; + let end = range.as_ref().map_or(size, |r| r.end); + let start = range.as_ref().map_or(0, |r| r.start); + let this = self.clone(); + let key_owned = key.to_owned(); + let st = St { inner: first, pos: start, @@ -446,7 +465,7 @@ impl BulkHttp { pub(crate) fn for_tests(endpoint: String, bucket: String) -> Self { BulkHttp { clients: vec![reqwest::Client::new()], - next: Default::default(), + next: std::sync::Arc::default(), creds: None, bucket, permits: std::sync::Arc::new(tokio::sync::Semaphore::new(8)), @@ -489,7 +508,12 @@ impl BulkHttp { .map(|g| format!("&ifGenerationMatch={g}")) .unwrap_or_default() ); - let mut req = self.clients[i].get(&url).headers(headers); + let mut req = self + .clients + .get(i) + .ok_or_else(|| StoreError::other(anyhow::anyhow!("empty bulk HTTP client pool")))? + .get(&url) + .headers(headers); if let Some(r) = &range { req = req.header( reqwest::header::RANGE, @@ -591,13 +615,12 @@ impl GcsStore { let mut builder = client.write_object(self.bucket_resource.clone(), key.to_owned(), b); builder = apply_put_opts(builder, &opts); - builder.send_unbuffered().await + Box::pin(builder.send_unbuffered()).await } PutBody::File(path) => { let small = tokio::fs::metadata(&path) .await - .map(|m| m.len() <= SINGLE_SHOT_PUT_LIMIT) - .unwrap_or(false); + .is_ok_and(|m| m.len() <= SINGLE_SHOT_PUT_LIMIT); if small { let bytes = tokio::fs::read(&path).await.map_err(StoreError::other)?; let (client, _permit) = self.data_client(key, false).await; @@ -607,7 +630,7 @@ impl GcsStore { Bytes::from(bytes), ); builder = apply_put_opts(builder, &opts); - builder.send_unbuffered().await + Box::pin(builder.send_unbuffered()).await } else { let stream = crate::util::file_stream(path, None, FILE_CHUNK_SIZE); let source = StoreStreamSource { @@ -617,16 +640,18 @@ impl GcsStore { let mut builder = client.write_object(self.bucket_resource.clone(), key.to_owned(), source); builder = apply_put_opts(builder, &opts); - builder.send_buffered().await + Box::pin(builder.send_buffered()).await } } PutBody::Stream { len, stream } if len <= SINGLE_SHOT_PUT_LIMIT => { - let bytes = crate::util::collect(stream, len as usize).await?; + let bytes = + crate::util::collect(stream, usize::try_from(len).map_err(StoreError::other)?) + .await?; let (client, _permit) = self.data_client(key, false).await; let mut builder = client.write_object(self.bucket_resource.clone(), key.to_owned(), bytes); builder = apply_put_opts(builder, &opts); - builder.send_unbuffered().await + Box::pin(builder.send_unbuffered()).await } PutBody::Stream { stream, .. } => { let source = StoreStreamSource { @@ -636,7 +661,7 @@ impl GcsStore { let mut builder = client.write_object(self.bucket_resource.clone(), key.to_owned(), source); builder = apply_put_opts(builder, &opts); - builder.send_buffered().await + Box::pin(builder.send_buffered()).await } }; @@ -662,40 +687,36 @@ impl ObjectStore for GcsStore { // match the current generation → the object is always "changed" // from the caller's perspective. Skip the precondition and return // the object directly. - match parse_generation(v) { - Some(generation) => { - let req = google_cloud_storage::model::GetObjectRequest::new() - .set_bucket(self.bucket_resource.clone()) - .set_object(key.to_owned()) - .set_if_generation_not_match(generation); - - let result = match call("get", key, META_DEADLINE, READ_RETRIES, || { - self.control.get_object().with_request(req.clone()).send() - }) - .await - { - Ok(obj) => { - let meta = Self::meta_from_object(&obj); - let body = self.read_object_body(key, opts.range.clone()).await?; - Ok(GetResult::Object { meta, body }) - } - Err(e) => { - if e.is_not_modified() { - Ok(GetResult::NotModified { - version: gen_version(generation), - }) - } else { - Err(e.into_store("get", key)) - } + if let Some(generation) = parse_generation(v) { + let req = google_cloud_storage::model::GetObjectRequest::new() + .set_bucket(self.bucket_resource.clone()) + .set_object(key.to_owned()) + .set_if_generation_not_match(generation); + + let result = match call("get", key, META_DEADLINE, READ_RETRIES, || { + self.control.get_object().with_request(req.clone()).send() + }) + .await + { + Ok(obj) => { + let meta = Self::meta_from_object(&obj); + let body = self.read_object_body(key, opts.range.clone()).await?; + Ok(GetResult::Object { meta, body }) + } + Err(e) => { + if e.is_not_modified() { + Ok(GetResult::NotModified { + version: gen_version(generation), + }) + } else { + Err(e.into_store("get", key)) } - }; - return result; - } - None => { - // Non-numeric version: can never match a GCS generation, - // so the object is always "changed" → fall through to read. - } + } + }; + return result; } + // Non-numeric version: can never match a GCS generation, + // so the object is always "changed" → fall through to read. } // Direct read (no if_none_match, or if_none_match with non-numeric @@ -718,9 +739,7 @@ impl ObjectStore for GcsStore { let meta = ObjectMeta { key: key.to_owned(), size, - version: generation - .map(gen_version) - .unwrap_or_else(|| Version::new("")), + version: generation.map_or_else(|| Version::new(""), gen_version), }; return Ok(GetResult::Object { meta, body }); } @@ -750,7 +769,7 @@ impl ObjectStore for GcsStore { let obj = resp.object(); let meta = ObjectMeta { key: key.to_owned(), - size: obj.size as u64, + size: obj.size.max(0).cast_unsigned(), version: gen_version(obj.generation), }; @@ -778,11 +797,11 @@ impl ObjectStore for GcsStore { async fn put(&self, key: &str, body: PutBody, opts: PutOptions) -> Result { let size_hint = match &body { PutBody::Bytes(b) => b.len() as u64, - PutBody::File(p) => tokio::fs::metadata(p).await.map(|m| m.len()).unwrap_or(0), + PutBody::File(p) => tokio::fs::metadata(p).await.map_or(0, |m| m.len()), PutBody::Stream { len, .. } => *len, }; let deadline = put_deadline(size_hint); - match tokio::time::timeout(deadline, self.put_inner(key, body, opts)).await { + match tokio::time::timeout(deadline, Box::pin(self.put_inner(key, body, opts))).await { Ok(r) => r, Err(_) => Err(deadline_error("put", key, deadline)), } @@ -854,24 +873,20 @@ impl ObjectStore for GcsStore { .set_object(key.to_owned()); if let Some(v) = &if_version { - match parse_generation(v) { - Some(generation) => { - req = req.set_if_generation_match(generation); - } - None => { - // Non-numeric version can never match a GCS generation. - // If the object exists → PreconditionFailed; else → NotFound. - if let Some(current) = self.current_generation(key).await { - return Err(StoreError::PreconditionFailed { - key: key.to_owned(), - current: Some(current), - }); - } else { - return Err(StoreError::NotFound { - key: key.to_owned(), - }); - } + if let Some(generation) = parse_generation(v) { + req = req.set_if_generation_match(generation); + } else { + // Non-numeric version can never match a GCS generation. + // If the object exists → PreconditionFailed; else → NotFound. + if let Some(current) = self.current_generation(key).await { + return Err(StoreError::PreconditionFailed { + key: key.to_owned(), + current: Some(current), + }); } + return Err(StoreError::NotFound { + key: key.to_owned(), + }); } } @@ -906,7 +921,7 @@ impl ObjectStore for GcsStore { let control = self.control.clone(); let bucket_resource = self.bucket_resource.clone(); let prefix = prefix.to_owned(); - let start_after = start_after.map(|s| s.to_owned()); + let start_after = start_after.map(std::borrow::ToOwned::to_owned); tokio::spawn(async move { let mut page_token = String::new(); @@ -942,10 +957,10 @@ impl ObjectStore for GcsStore { }; for obj in &resp.objects { - if let Some(ref sa) = skip_key { - if obj.name == *sa { - continue; - } + if let Some(ref sa) = skip_key + && obj.name == *sa + { + continue; } if tx.send(Ok(Self::meta_from_object(obj))).await.is_err() { return; // consumer dropped @@ -1007,7 +1022,7 @@ impl ObjectStore for GcsStore { let authorization = headers .get(http::header::AUTHORIZATION) .and_then(|v| v.to_str().ok()) - .map(|s| s.to_string()); + .map(std::string::ToString::to_string); Some(crate::AccelTarget { url: format!( "https://storage.googleapis.com/{}/{}", @@ -1074,10 +1089,7 @@ fn unfold_response( let stream = futures::stream::unfold( (Some(resp), key, permit), |(mut resp, key, permit)| async move { - let r = match resp.as_mut() { - Some(r) => r, - None => return None, - }; + let r = resp.as_mut()?; match tokio::time::timeout(READ_CHUNK_DEADLINE, r.next()).await { Ok(Some(Ok(bytes))) => Some((Ok(bytes), (resp, key, permit))), Ok(Some(Err(e))) => Some((Err(StoreError::other(e)), (resp, key, permit))), @@ -1144,10 +1156,10 @@ where // ---- error mapping ---- fn is_not_found(e: &google_cloud_storage::Error) -> bool { - if let Some(status) = e.status() { - if status.code == Code::NotFound { - return true; - } + if let Some(status) = e.status() + && status.code == Code::NotFound + { + return true; } if let Some(code) = e.http_status_code() { return code == 404; @@ -1158,10 +1170,10 @@ fn is_not_found(e: &google_cloud_storage::Error) -> bool { /// 304 Not Modified: returned when `if_generation_not_match` fails /// (generation IS the same → object unchanged). fn is_not_modified(e: &google_cloud_storage::Error) -> bool { - if let Some(code) = e.http_status_code() { - if code == 304 { - return true; - } + if let Some(code) = e.http_status_code() + && code == 304 + { + return true; } if let Some(status) = e.status() { // GCS returns 304 as FailedPrecondition for if_generation_not_match. @@ -1171,10 +1183,10 @@ fn is_not_modified(e: &google_cloud_storage::Error) -> bool { } fn is_precondition_failed(e: &google_cloud_storage::Error) -> bool { - if let Some(status) = e.status() { - if status.code == Code::FailedPrecondition { - return true; - } + if let Some(status) = e.status() + && status.code == Code::FailedPrecondition + { + return true; } if let Some(code) = e.http_status_code() { // GCS JSON API sometimes returns 412 with Code::Unknown. @@ -1223,7 +1235,7 @@ mod tests { #[test] fn gen_version_formats_decimal() { - assert_eq!(gen_version(1234567890).as_str(), "1234567890"); + assert_eq!(gen_version(1_234_567_890).as_str(), "1234567890"); assert_eq!(gen_version(0).as_str(), "0"); assert_eq!(gen_version(-1).as_str(), "-1"); } @@ -1460,9 +1472,11 @@ fn urlencode(s: &str) -> String { for b in s.bytes() { match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { - out.push(b as char) + out.push(b as char); + } + _ => { + let _ = std::fmt::Write::write_fmt(&mut out, format_args!("%{b:02X}")); } - _ => out.push_str(&format!("%{b:02X}")), } } out @@ -1534,14 +1548,13 @@ mod resume_tests { .and_then(|v| v.to_str().ok()) .and_then(|v| v.strip_prefix("bytes=")) .and_then(|v| v.split_once('-')) - .map(|(a, b)| { + .map_or((0, d.len()), |(a, b)| { (a.parse::().unwrap(), b.parse::().unwrap() + 1) - }) - .unwrap_or((0, d.len())); + }); let body: Vec = d[start..end].to_vec(); let cut = n < 2; let stream = futures::stream::iter( - body.chunks(100).map(|c| c.to_vec()).collect::>(), + body.chunks(100).map(<[u8]>::to_vec).collect::>(), ) .enumerate() .then(move |(i, c)| async move { diff --git a/crates/walgit-store/src/lib.rs b/crates/walgit-store/src/lib.rs index 05f1740..824196b 100644 --- a/crates/walgit-store/src/lib.rs +++ b/crates/walgit-store/src/lib.rs @@ -7,7 +7,7 @@ //! * conditional writes (`Create` = if-absent, `Update(v)` = CAS on version), //! * conditional deletes, range reads, streaming bodies, prefix listing. //! -//! [`Version`] is opaque to callers: GCS generation, S3/rustfs ETag, or a +//! [`Version`] is opaque to callers: GCS generation, S3/rustfs `ETag`, or a //! counter in [`memory::MemoryStore`]. Callers must never parse it. use std::{fmt, ops::Range, pin::Pin, sync::Arc}; @@ -29,7 +29,7 @@ pub mod util; pub type BoxStream<'a, T> = Pin + Send + 'a>>; pub type ByteStream = BoxStream<'static, Result>; -/// Opaque object version (GCS generation / ETag / counter). Compare only for equality. +/// Opaque object version (GCS generation / `ETag` / counter). Compare only for equality. #[derive(Clone, PartialEq, Eq, Hash)] pub struct Version(Arc); @@ -91,7 +91,8 @@ impl GetResult { match self { GetResult::NotModified { .. } => Ok(None), GetResult::Object { meta, body } => { - let b = util::collect(body, meta.size as usize).await?; + let b = util::collect(body, usize::try_from(meta.size).map_err(StoreError::other)?) + .await?; Ok(Some((meta, b))) } } @@ -104,9 +105,10 @@ impl GetResult { } } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Default)] pub enum PutMode { /// Unconditional overwrite. + #[default] Overwrite, /// Only if the object does not exist (if-generation-match: 0 / If-None-Match: *). Create, @@ -145,11 +147,6 @@ pub struct PutOptions { /// set long cache headers. pub immutable: bool, } -impl Default for PutMode { - fn default() -> Self { - PutMode::Overwrite - } -} impl From for PutOptions { fn from(mode: PutMode) -> Self { PutOptions { @@ -370,9 +367,7 @@ impl ObjectStore for Prefixed { let instrument = !self.inner.is_prefixed(); let full_key = self.full(key); // No span at all for nested prefix layers (avoids duplicate lines). - let span = if !instrument { - tracing::Span::none() - } else { + let span = if instrument { tracing::info_span!( "store.get", backend = self.inner.backend(), @@ -382,6 +377,8 @@ impl ObjectStore for Prefixed { outcome = tracing::field::Empty, error = tracing::field::Empty, ) + } else { + tracing::Span::none() }; let result = if instrument { self.inner @@ -425,9 +422,7 @@ impl ObjectStore for Prefixed { let instrument = !self.inner.is_prefixed(); let full_key = self.full(key); // No span at all for nested prefix layers (avoids duplicate lines). - let span = if !instrument { - tracing::Span::none() - } else { + let span = if instrument { tracing::info_span!( "store.head", backend = self.inner.backend(), @@ -437,6 +432,8 @@ impl ObjectStore for Prefixed { outcome = tracing::field::Empty, error = tracing::field::Empty, ) + } else { + tracing::Span::none() }; let result = if instrument { self.inner.head(&full_key).instrument(span.clone()).await @@ -470,14 +467,12 @@ impl ObjectStore for Prefixed { let bytes = match &body { PutBody::Bytes(b) => b.len() as u64, PutBody::Stream { len, .. } => *len, - PutBody::File(p) => std::fs::metadata(p).map(|m| m.len()).unwrap_or(0), + PutBody::File(p) => std::fs::metadata(p).map_or(0, |m| m.len()), }; let instrument = !self.inner.is_prefixed(); let full_key = self.full(key); // No span at all for nested prefix layers (avoids duplicate lines). - let span = if !instrument { - tracing::Span::none() - } else { + let span = if instrument { tracing::info_span!( "store.put", backend = self.inner.backend(), @@ -486,6 +481,8 @@ impl ObjectStore for Prefixed { outcome = tracing::field::Empty, error = tracing::field::Empty, ) + } else { + tracing::Span::none() }; let result = if instrument { self.inner @@ -518,9 +515,7 @@ impl ObjectStore for Prefixed { let instrument = !self.inner.is_prefixed(); let full_key = self.full(key); // No span at all for nested prefix layers (avoids duplicate lines). - let span = if !instrument { - tracing::Span::none() - } else { + let span = if instrument { tracing::info_span!( "store.delete", backend = self.inner.backend(), @@ -528,6 +523,8 @@ impl ObjectStore for Prefixed { outcome = tracing::field::Empty, error = tracing::field::Empty, ) + } else { + tracing::Span::none() }; let result = if instrument { self.inner @@ -561,6 +558,8 @@ impl ObjectStore for Prefixed { prefix: &str, start_after: Option<&str>, ) -> BoxStream<'static, Result> { + use futures::StreamExt; + let full_prefix = self.full(prefix); let _span = (!self.inner.is_prefixed()).then(|| { tracing::debug_span!( @@ -570,7 +569,7 @@ impl ObjectStore for Prefixed { ) .entered() }); - use futures::StreamExt; + let this = self.clone(); let start_after = start_after.map(|s| self.full(s)); Box::pin( @@ -637,7 +636,7 @@ pub async fn open_store(cfg: &walgit_config::Config) -> anyhow::Result walgit_config::StoreBackend::S3 => { #[cfg(feature = "s3")] { - Arc::new(s3::S3Store::new(&cfg.store).await?) + Arc::new(s3::S3Store::new(&cfg.store)?) } #[cfg(not(feature = "s3"))] { diff --git a/crates/walgit-store/src/memory.rs b/crates/walgit-store/src/memory.rs index 764a52e..e786d9b 100644 --- a/crates/walgit-store/src/memory.rs +++ b/crates/walgit-store/src/memory.rs @@ -63,7 +63,9 @@ impl MemoryStore { async fn body_bytes(body: PutBody) -> Result { Ok(match body { PutBody::Bytes(b) => b, - PutBody::Stream { len, stream } => util::collect(stream, len as usize).await?, + PutBody::Stream { len, stream } => { + util::collect(stream, usize::try_from(len).map_err(StoreError::other)?).await? + } PutBody::File(p) => Bytes::from(tokio::fs::read(&p).await.map_err(StoreError::other)?), }) } @@ -115,8 +117,8 @@ impl ObjectStore for MemoryStore { let size = data.len() as u64; let slice = match &opts.range { Some(r) => { - let start = r.start.min(size) as usize; - let end = r.end.min(size) as usize; + let start = usize::try_from(r.start.min(size)).map_err(StoreError::other)?; + let end = usize::try_from(r.end.min(size)).map_err(StoreError::other)?; if start > end { return Err(StoreError::InvalidArgument(format!( "bad range {r:?} for size {size}" @@ -151,8 +153,7 @@ impl ObjectStore for MemoryStore { let mut g = self.objects.lock(); let current = g.get(key).map(|(v, _)| v.clone()); match (&opts.mode, ¤t) { - (PutMode::Overwrite, _) => {} - (PutMode::Create, None) => {} + (PutMode::Overwrite, _) | (PutMode::Create, None) => {} (PutMode::Create, Some(v)) => { return Err(StoreError::PreconditionFailed { key: key.into(), @@ -245,8 +246,9 @@ impl ObjectStore for MemoryStore { .range(prefix.to_owned()..) .take_while(|(k, _)| k.starts_with(prefix)) .filter_map(|(k, _)| { - let rest = &k[prefix.len()..]; - rest.find('/').map(|i| format!("{prefix}{}/", &rest[..i])) + let rest = k.strip_prefix(prefix)?; + rest.split_once('/') + .map(|(head, _)| format!("{prefix}{head}/")) }) .collect(); out.dedup(); diff --git a/crates/walgit-store/src/s3.rs b/crates/walgit-store/src/s3.rs index 2877ae2..878aa08 100644 --- a/crates/walgit-store/src/s3.rs +++ b/crates/walgit-store/src/s3.rs @@ -7,29 +7,29 @@ //! //! ## Version tokens //! -//! S3 ETags are used as opaque `Version` strings. Quotes are stripped +//! S3 `ETags` are used as opaque `Version` strings. Quotes are stripped //! consistently on read and never stored. For non-multipart uploads the -//! ETag is the MD5 of the content; for multipart uploads it is a compound +//! `ETag` is the MD5 of the content; for multipart uploads it is a compound //! hash. Callers never parse the token — equality comparison suffices. //! //! ## Conditional PUT //! //! `PutMode::Create` → `If-None-Match: *` (object must not exist). -//! `PutMode::Update(v)` → `If-Match: ` (CAS on current ETag). +//! `PutMode::Update(v)` → `If-Match: ` (CAS on current `ETag`). //! On failure the SDK returns a `PreconditionFailed` service error; we fill //! `current` via a follow-up HEAD when the SDK doesn't include it. //! //! ## Conditional DELETE //! -//! S3 has no native conditional delete. We emulate via HEAD (read ETag) + +//! S3 has no native conditional delete. We emulate via HEAD (read `ETag`) + //! compare + DELETE, documenting the inherent check-then-act race: a //! concurrent writer could replace the object between HEAD and DELETE. //! Acceptable for walgit's lease-guarded semantics. //! //! ## Multipart upload //! -//! Objects above `cfg.multipart_threshold` use CreateMultipartUpload + -//! UploadPart + CompleteMultipartUpload. CreateMultipartUpload does NOT +//! Objects above `cfg.multipart_threshold` use `CreateMultipartUpload` + +//! `UploadPart` + `CompleteMultipartUpload`. `CreateMultipartUpload` does NOT //! support `If-None-Match`/`If-Match` in the S3 API, so multipart is only //! used for `PutMode::Overwrite`. For walgit's immutable pack objects //! (`PutMode::Create`) we use single-shot PUT when the object is large, @@ -73,7 +73,7 @@ impl S3Store { /// `cfg.s3.access_key_env` / `cfg.s3.secret_key_env` /// (defaults `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`), plus /// `AWS_SESSION_TOKEN` when present. - pub async fn new(cfg: &walgit_config::StoreConfig) -> anyhow::Result { + pub fn new(cfg: &walgit_config::StoreConfig) -> anyhow::Result { let access_key = std::env::var(&cfg.s3.access_key_env).map_err(|_| { anyhow::anyhow!("s3: env var {} not set (access key)", cfg.s3.access_key_env) })?; @@ -118,7 +118,7 @@ impl S3Store { // ---- GET via presigned URL + reqwest (true streaming) --------------- async fn presigned_get(&self, key: &str, opts: &GetOptions) -> Result { - let presigning = PresigningConfig::expires_in(Duration::from_secs(60)) + let presigning = PresigningConfig::expires_in(Duration::from_mins(1)) .map_err(|e| StoreError::other(anyhow::anyhow!("presigning config: {e}")))?; let mut builder = self.client.get_object().bucket(&self.bucket).key(key); @@ -190,7 +190,7 @@ impl S3Store { 404 => Err(StoreError::NotFound { key: key.into() }), 412 => Err(StoreError::PreconditionFailed { key: key.into(), - current: etag.map(|e| Version::new(e)), + current: etag.map(Version::new), }), s if s >= 500 || s == 429 => { Err(StoreError::Retryable(anyhow::anyhow!("s3 get status {s}"))) @@ -212,7 +212,8 @@ async fn body_to_s3(body: PutBody) -> Result<(S3ByteStream, u64)> { // Collect into Bytes: walgit's Stream bodies are small objects // (manifests, leases). Large packs use PutBody::File which // streams via ByteStream::read_from(). - let collected = util::collect(stream, len as usize).await?; + let collected = + util::collect(stream, usize::try_from(len).map_err(StoreError::other)?).await?; (S3ByteStream::from(collected), len) } PutBody::File(path) => { @@ -233,19 +234,19 @@ async fn body_to_s3(body: PutBody) -> Result<(S3ByteStream, u64)> { // ---- error classification ---------------------------------------------- -/// Extract the error code string from an SdkError's service error metadata. +/// Extract the error code string from an `SdkError`'s service error metadata. fn err_code(err: &aws_sdk_s3::error::SdkError) -> Option<&str> where E: aws_sdk_s3::error::ProvideErrorMetadata, { - err.as_service_error().map(|e| e.meta().code()).flatten() + err.as_service_error().and_then(|e| e.meta().code()) } fn classify_put_error( key: &str, - err: aws_sdk_s3::error::SdkError, + err: &aws_sdk_s3::error::SdkError, ) -> StoreError { - let code = err_code(&err).unwrap_or(""); + let code = err_code(err).unwrap_or(""); match code { "PreconditionFailed" | "ConditionalRequestConflict" => StoreError::PreconditionFailed { key: key.into(), @@ -256,7 +257,7 @@ fn classify_put_error( } fn classify_list_error( - err: aws_sdk_s3::error::SdkError, + err: &aws_sdk_s3::error::SdkError, ) -> StoreError { StoreError::Other(anyhow::anyhow!("s3 list error: {err}")) } @@ -284,7 +285,8 @@ impl ObjectStore for S3Store { match resp { Ok(out) => { let etag = out.e_tag().map(|s| s.trim_matches('"').to_owned()); - let size = out.content_length().unwrap_or(0) as u64; + let size = + u64::try_from(out.content_length().unwrap_or(0)).map_err(StoreError::other)?; Ok(Some(ObjectMeta { key: key.into(), size, @@ -321,7 +323,7 @@ impl ObjectStore for S3Store { .bucket(&self.bucket) .key(key) .body(s3_body) - .content_length(len as i64); + .content_length(i64::try_from(len).map_err(StoreError::other)?); match &opts.mode { PutMode::Overwrite => {} @@ -348,12 +350,12 @@ impl ObjectStore for S3Store { }) } Err(e) => { - let mut err = classify_put_error(key, e); + let mut err = classify_put_error(key, &e); // Fill `current` via HEAD if we got a PreconditionFailed. - if let StoreError::PreconditionFailed { current: c, .. } = &mut err { - if c.is_none() { - *c = self.head(key).await.ok().flatten().map(|m| m.version); - } + if let StoreError::PreconditionFailed { current: c, .. } = &mut err + && c.is_none() + { + *c = self.head(key).await.ok().flatten().map(|m| m.version); } Err(err) } @@ -415,7 +417,7 @@ impl ObjectStore for S3Store { let client = self.client.clone(); let bucket = self.bucket.clone(); let prefix = prefix.to_owned(); - let start_after = start_after.map(|s| s.to_owned()); + let start_after = start_after.map(std::borrow::ToOwned::to_owned); Box::pin(futures::stream::unfold( ListState { @@ -461,7 +463,8 @@ impl ObjectStore for S3Store { let etag = obj.e_tag().map(|s| s.trim_matches('"').to_owned()); Ok(ObjectMeta { key: obj.key().unwrap_or("").to_owned(), - size: obj.size().unwrap_or(0) as u64, + size: u64::try_from(obj.size().unwrap_or(0)) + .map_err(StoreError::other)?, version: Version::new(etag.as_deref().unwrap_or("")), }) }) @@ -470,14 +473,17 @@ impl ObjectStore for S3Store { state.continuation_token = resp .is_truncated() .unwrap_or(false) - .then(|| resp.next_continuation_token().map(|s| s.to_owned())) + .then(|| { + resp.next_continuation_token() + .map(std::borrow::ToOwned::to_owned) + }) .flatten(); state.buffer = items.into_iter(); let item = state.buffer.next(); item.map(|i| (i, state)) } - Err(err) => Some((Err(classify_list_error(err)), state)), + Err(err) => Some((Err(classify_list_error(&err)), state)), } }, )) @@ -497,7 +503,7 @@ impl ObjectStore for S3Store { if let Some(ct) = &continuation_token { builder = builder.continuation_token(ct); } - let resp = builder.send().await.map_err(classify_list_error)?; + let resp = builder.send().await.map_err(|e| classify_list_error(&e))?; out.extend( resp.common_prefixes() .iter() @@ -506,7 +512,10 @@ impl ObjectStore for S3Store { continuation_token = resp .is_truncated() .unwrap_or(false) - .then(|| resp.next_continuation_token().map(|s| s.to_owned())) + .then(|| { + resp.next_continuation_token() + .map(std::borrow::ToOwned::to_owned) + }) .flatten(); if continuation_token.is_none() { break; @@ -520,7 +529,7 @@ impl ObjectStore for S3Store { /// A presigned GET (1 h): the edge needs no credentials and `Range` stays free (unsigned). async fn accel_target(&self, key: &str) -> Option { let url = self - .signed_get_url(key, Duration::from_secs(3600)) + .signed_get_url(key, Duration::from_hours(1)) .await .ok() .flatten()?; @@ -569,7 +578,15 @@ impl ObjectStore for S3Store { .ok_or_else(|| StoreError::NotFound { key: src.clone() })?; sizes.push(m.size); } - let total: u64 = sizes.iter().sum(); + let mut total = 0u64; + let mut layout = Vec::with_capacity(sources.len()); + for (source, size) in sources.iter().zip(&sizes) { + let end = total + .checked_add(*size) + .ok_or_else(|| StoreError::other(anyhow::anyhow!("compose size overflow")))?; + layout.push((total, end, source)); + total = end; + } // The virtual concatenation, cut into parts: a part is [start, end) of the whole. // Runs that lie inside one source and are >= MIN_PART become copies; everything else // (a small source, the tail that pads it to MIN_PART) is read and uploaded. @@ -597,20 +614,24 @@ impl ObjectStore for S3Store { let mut parts: Vec = Vec::new(); let mut part_number = 1i32; let mut pos: u64 = 0; // absolute offset into the concatenation - let offset_of = |i: usize| -> u64 { sizes[..i].iter().sum() }; + let source_at = |position| { + layout + .iter() + .find(|(_, end, _)| position < *end) + .ok_or_else(|| { + StoreError::other(anyhow::anyhow!("compose source offset out of bounds")) + }) + }; let result: Result<()> = async { while pos < total { // Which source does `pos` fall in, and how far does it run? - let i = (0..sources.len()) - .find(|&i| pos < offset_of(i) + sizes[i]) - .unwrap(); - let src_end = offset_of(i) + sizes[i]; + let &(src_start, src_end, source) = source_at(pos)?; let run = src_end - pos; let last_part = src_end == total; if run >= MIN_PART || last_part { // Copy a range of this one source. let len = run.min(COPY_PART); - let from = pos - offset_of(i); + let from = pos - src_start; let part = self .client .upload_part_copy() @@ -621,7 +642,7 @@ impl ObjectStore for S3Store { .copy_source(format!( "{}/{}", self.bucket, - crate::util::encode_path(&sources[i]) + crate::util::encode_path(source) )) .copy_source_range(format!("bytes={from}-{}", from + len - 1)) .send() @@ -644,17 +665,16 @@ impl ObjectStore for S3Store { } else { // Too small to copy on its own: read MIN_PART bytes across source boundaries. let want = MIN_PART.min(total - pos); - let mut buf = Vec::with_capacity(want as usize); + let mut buf = + Vec::with_capacity(usize::try_from(want).map_err(StoreError::other)?); let mut p = pos; while (buf.len() as u64) < want { - let j = (0..sources.len()) - .find(|&j| p < offset_of(j) + sizes[j]) - .unwrap(); - let from = p - offset_of(j); - let take = (sizes[j] - from).min(want - buf.len() as u64); + let &(source_start, source_end, source) = source_at(p)?; + let from = p - source_start; + let take = (source_end - p).min(want - buf.len() as u64); let (_, bytes) = self .get( - &sources[j], + source, GetOptions { range: Some(from..from + take), ..GetOptions::default() @@ -664,7 +684,7 @@ impl ObjectStore for S3Store { .bytes() .await? .ok_or_else(|| StoreError::NotFound { - key: sources[j].clone(), + key: source.clone(), })?; buf.extend_from_slice(&bytes); p += take; @@ -678,7 +698,7 @@ impl ObjectStore for S3Store { .upload_id(&upload_id) .part_number(part_number) .body(S3ByteStream::from(Bytes::from(buf))) - .content_length(len as i64) + .content_length(i64::try_from(len).map_err(StoreError::other)?) .send() .await .map_err(|e| StoreError::Other(anyhow::anyhow!("s3 upload part: {e}")))?; @@ -764,6 +784,8 @@ impl S3Store { len: u64, opts: &PutOptions, ) -> Result { + use tokio::io::AsyncReadExt; + let mut create = self .client .create_multipart_upload() @@ -791,17 +813,21 @@ impl S3Store { let mut uploaded_parts: Vec = Vec::new(); let mut remaining = len; - use tokio::io::AsyncReadExt; let mut reader = body.into_async_read(); while remaining > 0 { let this_part = part_size.min(remaining); - let to_read = this_part as usize; + let to_read = usize::try_from(this_part).map_err(StoreError::other)?; let mut buf = vec![0u8; to_read]; let mut read_total = 0; while read_total < to_read { - let n = match reader.read(&mut buf[read_total..]).await { + let n = match reader + .read(buf.get_mut(read_total..).ok_or_else(|| { + StoreError::other(anyhow::anyhow!("multipart read exceeded buffer")) + })?) + .await + { Ok(n) => n, Err(e) => { let _ = self.abort_multipart(key, &upload_id).await; @@ -828,7 +854,7 @@ impl S3Store { .upload_id(&upload_id) .part_number(part_number) .body(S3ByteStream::from(Bytes::from(buf))) - .content_length(actual as i64) + .content_length(i64::try_from(actual).map_err(StoreError::other)?) .send() .await { diff --git a/crates/walgit-store/src/util.rs b/crates/walgit-store/src/util.rs index f4cc23b..b99cc1f 100644 --- a/crates/walgit-store/src/util.rs +++ b/crates/walgit-store/src/util.rs @@ -11,10 +11,9 @@ pub async fn collect(mut body: ByteStream, size_hint: usize) -> Result { let chunk = chunk?; match (&mut first, &mut buf) { (None, None) => first = Some(chunk), - (Some(_), None) => { - let f = first.take().unwrap(); + (Some(f), None) => { let mut b = BytesMut::with_capacity(size_hint.max(f.len() + chunk.len())); - b.extend_from_slice(&f); + b.extend_from_slice(f); b.extend_from_slice(&chunk); buf = Some(b); } @@ -40,9 +39,6 @@ pub fn file_stream( chunk: usize, ) -> ByteStream { use tokio::io::{AsyncReadExt, AsyncSeekExt}; - return async_stream_file(path, range, chunk) - .map(|r| r.map_err(StoreError::other)) - .boxed(); fn async_stream_file( path: std::path::PathBuf, @@ -63,10 +59,10 @@ pub fn file_stream( Err(e) => return Some((Err(e), State::Done)), }, }; - if start > 0 { - if let Err(e) = f.seek(std::io::SeekFrom::Start(start)).await { - return Some((Err(e), State::Done)); - } + if start > 0 + && let Err(e) = f.seek(std::io::SeekFrom::Start(start)).await + { + return Some((Err(e), State::Done)); } read_next(f, remaining, chunk).await } @@ -87,7 +83,7 @@ pub fn file_stream( if remaining == 0 { return None; } - let want = (chunk as u64).min(remaining) as usize; + let want = chunk.min(usize::try_from(remaining).unwrap_or(usize::MAX)); let mut buf = BytesMut::with_capacity(want); // read_buf reads at most capacity; loop until we get `want` or EOF. while buf.len() < want { @@ -123,6 +119,9 @@ pub fn file_stream( }, Done, } + async_stream_file(path, range, chunk) + .map(|r| r.map_err(StoreError::other)) + .boxed() } /// Exponential backoff with full jitter. `attempt` starts at 0. @@ -134,7 +133,7 @@ pub fn backoff( use rand::Rng; let exp = base.saturating_mul(1u32 << attempt.min(16)); let cap = exp.min(max); - let jitter = rand::rng().random_range(0..=cap.as_millis() as u64); + let jitter = rand::rng().random_range(0..=u64::try_from(cap.as_millis()).unwrap_or(u64::MAX)); std::time::Duration::from_millis(jitter) } @@ -167,6 +166,10 @@ where /// compose natively (S3 does its own multipart PUT) or the file is small. Part objects live under `.part/NNNN` /// and are deleted afterwards (best effort). `opts.mode` applies to the final /// object only; a `Create` precondition failure surfaces as such. +#[expect( + clippy::cast_precision_loss, + reason = "Transfer throughput is approximate telemetry" +)] pub async fn put_file_parallel( store: &dyn crate::ObjectStore, key: &str, @@ -268,9 +271,11 @@ pub fn encode_path(key: &str) -> String { for b in key.bytes() { match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' | b'/' => { - out.push(b as char) + out.push(b as char); + } + _ => { + let _ = std::fmt::Write::write_fmt(&mut out, format_args!("%{b:02X}")); } - _ => out.push_str(&format!("%{b:02X}")), } } out diff --git a/crates/walgit-store/tests/contract.rs b/crates/walgit-store/tests/contract.rs index 1db260e..768b860 100644 --- a/crates/walgit-store/tests/contract.rs +++ b/crates/walgit-store/tests/contract.rs @@ -1,3 +1,14 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow( + clippy::expect_used, + clippy::indexing_slicing, + clippy::panic, + clippy::unwrap_used, + clippy::cast_possible_truncation, + clippy::cast_precision_loss, + clippy::match_wildcard_for_single_variants +)] + //! Backend-agnostic contract suite for `ObjectStore`. //! //! `run_contract(store, prefix)` exercises every observable guarantee of the @@ -8,7 +19,7 @@ //! //! The suite is executed against `MemoryStore` always, and against `S3Store` //! when `WALGIT_TEST_S3_ENDPOINT` is set. `GcsStore` is tested when -//! `WALGIT_TEST_GCS_BUCKET` is set (StoreGcs adds that wrapper). +//! `WALGIT_TEST_GCS_BUCKET` is set (`StoreGcs` adds that wrapper). use std::ops::Range; use std::sync::Arc; @@ -117,7 +128,7 @@ async fn test_compose(store: &DynStore, key: &str) { // ---- helpers ----------------------------------------------------------- -/// Collect a GetResult body into Bytes, asserting it's an Object. +/// Collect a `GetResult` body into Bytes, asserting it's an Object. async fn collect_body(r: GetResult) -> (walgit_store::ObjectMeta, Bytes) { match r { GetResult::Object { meta, body } => { @@ -189,7 +200,7 @@ async fn test_put_create_wins_once(store: &DynStore, key: &str) { let _ = store.delete(key, None).await; } -/// Update CAS: winner updates, loser gets PreconditionFailed. +/// Update CAS: winner updates, loser gets `PreconditionFailed`. async fn test_update_cas(store: &DynStore, key: &str) { let _ = store.delete(key, None).await; @@ -266,7 +277,7 @@ async fn test_update_cas(store: &DynStore, key: &str) { let _ = store.delete(key, None).await; } -/// if_none_match: NotModified when unchanged, Object when changed. +/// `if_none_match`: `NotModified` when unchanged, Object when changed. async fn test_get_if_none_match(store: &DynStore, key: &str) { let _ = store.delete(key, None).await; @@ -306,7 +317,7 @@ async fn test_get_if_none_match(store: &DynStore, key: &str) { let _ = store.delete(key, None).await; } -/// if_match mismatch → PreconditionFailed. +/// `if_match` mismatch → `PreconditionFailed`. async fn test_get_if_match_mismatch(store: &DynStore, key: &str) { let _ = store.delete(key, None).await; @@ -463,7 +474,7 @@ async fn test_delete(store: &DynStore, key: &str) { ); } -/// list: ordering, start_after, prefix isolation. +/// list: ordering, `start_after`, prefix isolation. async fn test_list(store: &DynStore, base: &str) { // Clean up any previous data under base. let existing: Vec<_> = store.list(base, None).collect::>().await; @@ -555,6 +566,8 @@ async fn test_list(store: &DynStore, base: &str) { /// 8 MiB streamed put/get roundtrip with checksum. async fn test_large_streamed_roundtrip(store: &DynStore, key: &str) { + use sha1::{Digest, Sha1}; + let _ = store.delete(key, None).await; // 8 MiB of pseudo-random but deterministic data. @@ -571,7 +584,6 @@ async fn test_large_streamed_roundtrip(store: &DynStore, key: &str) { let data = Bytes::from(data); // Checksum (SHA-1). - use sha1::{Digest, Sha1}; let mut hasher = Sha1::new(); hasher.update(&data); let expected_checksum = hasher.finalize(); @@ -614,8 +626,8 @@ async fn test_large_streamed_roundtrip(store: &DynStore, key: &str) { } /// Multipart path: put an object above the threshold, verify roundtrip. -/// For MemoryStore this exercises the same code path (no multipart). -/// For S3Store with a small threshold, this triggers multipart upload. +/// For `MemoryStore` this exercises the same code path (no multipart). +/// For `S3Store` with a small threshold, this triggers multipart upload. async fn test_multipart_path(store: &DynStore, key: &str) { let _ = store.delete(key, None).await; @@ -667,12 +679,9 @@ async fn memory_contract() { #[cfg(feature = "s3")] #[tokio::test] async fn s3_contract() { - let endpoint = match std::env::var("WALGIT_TEST_S3_ENDPOINT") { - Ok(v) => v, - Err(_) => { - eprintln!("skipping s3_contract: WALGIT_TEST_S3_ENDPOINT not set"); - return; - } + let Ok(endpoint) = std::env::var("WALGIT_TEST_S3_ENDPOINT") else { + eprintln!("skipping s3_contract: WALGIT_TEST_S3_ENDPOINT not set"); + return; }; let bucket = std::env::var("WALGIT_TEST_BUCKET").unwrap_or_else(|_| "walgit-test".into()); let _access_key = @@ -699,9 +708,7 @@ async fn s3_contract() { ..Default::default() }; - let store = walgit_store::s3::S3Store::new(&cfg) - .await - .expect("S3Store::new"); + let store = walgit_store::s3::S3Store::new(&cfg).expect("S3Store::new"); let store: DynStore = Arc::new(store); run_contract(store.clone(), &prefix).await; @@ -723,12 +730,9 @@ async fn s3_contract() { #[cfg(feature = "gcs")] #[tokio::test] async fn gcs_contract() { - let bucket = match std::env::var("WALGIT_TEST_GCS_BUCKET") { - Ok(v) => v, - Err(_) => { - eprintln!("skipping gcs_contract: WALGIT_TEST_GCS_BUCKET not set"); - return; - } + let Ok(bucket) = std::env::var("WALGIT_TEST_GCS_BUCKET") else { + eprintln!("skipping gcs_contract: WALGIT_TEST_GCS_BUCKET not set"); + return; }; // Install the rustls crypto provider (required for TLS with google-cloud-storage). @@ -776,6 +780,8 @@ async fn gcs_contract() { /// stay under 2 s. `WALGIT_TEST_GCS_BUCKET=walgit-store WALGIT_TEST_GCS_BIG_KEY=`. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn gcs_control_plane_not_starved_by_bulk() { + const CHUNK: u64 = 32 * 1024 * 1024; + let (Ok(bucket), Ok(big_key)) = ( std::env::var("WALGIT_TEST_GCS_BUCKET"), std::env::var("WALGIT_TEST_GCS_BIG_KEY"), @@ -820,7 +826,7 @@ async fn gcs_control_plane_not_starved_by_bulk() { .await; eprintln!("baseline probe {:?}", t.elapsed()); let total = (1u64 << 30).min(size); - const CHUNK: u64 = 32 * 1024 * 1024; + let bulk = { let store = store.clone(); let big_key = big_key.clone(); diff --git a/crates/walgit-wal/src/checkpoint.rs b/crates/walgit-wal/src/checkpoint.rs index e1cfd4d..edad910 100644 --- a/crates/walgit-wal/src/checkpoint.rs +++ b/crates/walgit-wal/src/checkpoint.rs @@ -1,4 +1,5 @@ //! Checkpoint writing and store GC. +#![allow(clippy::needless_continue)] use std::sync::Arc; @@ -43,7 +44,7 @@ pub fn checkpoint_due( if head == 0 { return None; } - let cp_seq = manifest.checkpoint.as_ref().map(|c| c.seq).unwrap_or(0); + let cp_seq = manifest.checkpoint.as_ref().map_or(0, |c| c.seq); if cp_seq >= head { return None; } @@ -73,43 +74,38 @@ pub fn checkpoint_due( Some(c) => c.created_at.as_ref().map(time::to_system), None => manifest.updated_at.as_ref().map(time::to_system), }; - if let Some(t) = since { - if std::time::SystemTime::now() + if let Some(t) = since + && std::time::SystemTime::now() .duration_since(t) .unwrap_or_default() >= cfg.checkpoint_interval - { - return Some(CheckpointTrigger::Age); - } + { + return Some(CheckpointTrigger::Age); } } None } /// Write a checkpoint at the current head: refs snapshot + pack set, then -/// CAS manifest (checkpoint=, min_seq=, log_segments trimmed). Idempotent. +/// CAS manifest (checkpoint=, `min_seq`=, `log_segments` trimmed). Idempotent. /// Needs only a **refs-level** sync (manifest + ref state): it works on an /// instance that could never hold the repo's packs. pub(crate) async fn write_checkpoint_impl(handle: &RepoHandle) -> Result { let trigger = checkpoint_due(&handle.manifest(), &handle.cfg.wal) - .map(|t| t.to_string()) - .unwrap_or_else(|| "manual".into()); + .map_or_else(|| "manual".into(), |t| t.to_string()); let span = tracing::info_span!("wal.checkpoint", repo = %handle.id, trigger = %trigger, seq = tracing::field::Empty, refs = tracing::field::Empty, folded = tracing::field::Empty, outcome = tracing::field::Empty); let t0 = std::time::Instant::now(); let r = write_checkpoint_inner(handle) .instrument(span.clone()) .await; - match &r { - Ok(cp) => { - span.record("seq", cp.seq); - span.record("outcome", "ok"); - metrics::histogram!("walgit_checkpoint_seconds").record(t0.elapsed().as_secs_f64()); - metrics::counter!("walgit_checkpoints_total", "outcome" => "ok").increment(1); - } - Err(_) => { - span.record("outcome", "error"); - metrics::counter!("walgit_checkpoints_total", "outcome" => "error").increment(1); - } + if let Ok(cp) = &r { + span.record("seq", cp.seq); + span.record("outcome", "ok"); + metrics::histogram!("walgit_checkpoint_seconds").record(t0.elapsed().as_secs_f64()); + metrics::counter!("walgit_checkpoints_total", "outcome" => "ok").increment(1); + } else { + span.record("outcome", "error"); + metrics::counter!("walgit_checkpoints_total", "outcome" => "error").increment(1); } r } @@ -123,17 +119,17 @@ async fn write_checkpoint_inner(handle: &RepoHandle) -> Result Result Result Result= current_manifest.head_seq { - return Ok(cp.clone()); - } + if let Some(ref cp) = current_manifest.checkpoint + && cp.seq >= current_manifest.head_seq + { + return Ok(cp.clone()); } let mut updated: Manifest = (*current_manifest).clone(); @@ -234,7 +230,7 @@ async fn write_checkpoint_inner(handle: &RepoHandle) -> Result seq updated.log_segments.retain(|s| s.last_seq > seq); updated.updated_at = Some(time::now()); - updated.writer = writer.to_string(); + updated.writer = writer.clone(); updated.revision += 1; let buf = updated.encode_to_vec(); diff --git a/crates/walgit-wal/src/handle.rs b/crates/walgit-wal/src/handle.rs index 2f6acbb..6f9667e 100644 --- a/crates/walgit-wal/src/handle.rs +++ b/crates/walgit-wal/src/handle.rs @@ -1,4 +1,4 @@ -//! RepoHandle: per-repository state, sync, publish, checkpoint. +//! `RepoHandle`: per-repository state, sync, publish, checkpoint. use std::collections::HashMap; use std::sync::{ @@ -128,6 +128,10 @@ impl ObjectAccess { } impl RepoHandle { + #[expect( + clippy::too_many_arguments, + reason = "Repository construction combines the shared services and loaded WAL state" + )] pub(crate) fn new( id: RepoId, local: LocalRepo, @@ -266,10 +270,10 @@ impl RepoHandle { return Ok((guard, ObjectAccess::Local)); } // Remote: reuse the reader for this manifest revision, else (re)open. - if let Some(r) = self.remote.lock().clone() { - if r.revision == manifest.revision { - return Ok((guard, ObjectAccess::Remote(r))); - } + if let Some(r) = self.remote.lock().clone() + && r.revision == manifest.revision + { + return Ok((guard, ObjectAccess::Remote(r))); } let remote = self.open_remote(&manifest).await?; Ok((guard, ObjectAccess::Remote(remote))) @@ -322,7 +326,7 @@ impl RepoHandle { } Begin::AlreadyRunning(state) => { // Another request is opening it: wait for that task, then reuse. - let _ = state.wait_done(std::time::Duration::from_secs(600)).await; + let _ = state.wait_done(std::time::Duration::from_mins(10)).await; match state.outcome() { Some(Ok(_)) => {} Some(Err((_, m))) => { @@ -644,33 +648,30 @@ impl RepoHandle { // The whole materialization runs on the bulk runtime (own threads): // nothing in it can stall this runtime's request workers. let arc = self.self_arc.get().cloned(); - let res = match arc { - Some(arc) => { - let m = manifest.clone(); - let task_span = task.as_ref().map(|t| t.span()); - crate::sync::on_bulk_runtime(async move { - let work = async { - crate::sync::reconcile_packs(&arc, &m, level).await?; - arc.local.refresh_async().await?; - Ok::<(), WalError>(()) - }; - match task_span { - Some(sp) => work.instrument(sp).await, - None => work.await, - } - }) - .await - } - None => { - let res = async { - crate::sync::reconcile_packs(self, &manifest, level).await?; - self.local.refresh_async().await?; + let res = if let Some(arc) = arc { + let m = manifest.clone(); + let task_span = task.as_ref().map(super::tasks::TaskHandle::span); + crate::sync::on_bulk_runtime(async move { + let work = async { + crate::sync::reconcile_packs(&arc, &m, level).await?; + arc.local.refresh_async().await?; Ok::<(), WalError>(()) }; - match &task { - Some(t) => res.instrument(t.span()).await, - None => res.await, + match task_span { + Some(sp) => work.instrument(sp).await, + None => work.await, } + }) + .await + } else { + let res = async { + crate::sync::reconcile_packs(self, &manifest, level).await?; + self.local.refresh_async().await?; + Ok::<(), WalError>(()) + }; + match &task { + Some(t) => res.instrument(t.span()).await, + None => res.await, } }; *self.active_reporter.lock() = None; @@ -734,8 +735,10 @@ impl RepoHandle { .collect(); } let mount = self.mount_dir(); - if mount.is_none() && self.cfg.cache.store_mount.is_some() { - tracing::warn!(repo = %self.id, mount = %self.cfg.cache.store_mount.as_ref().unwrap().display(), "store mount configured but the repository directory is not visible in it (gcsfuse not up yet?): base packs served remotely until it is"); + if mount.is_none() + && let Some(store_mount) = &self.cfg.cache.store_mount + { + tracing::warn!(repo = %self.id, mount = %store_mount.display(), "store mount configured but the repository directory is not visible in it (gcsfuse not up yet?): base packs served remotely until it is"); } manifest .packs @@ -775,10 +778,10 @@ impl RepoHandle { /// `remote-index` task while opening. pub async fn remote_reader(&self) -> Result, WalError> { let manifest = self.manifest(); - if let Some(r) = self.remote.lock().clone() { - if r.revision == manifest.revision { - return Ok(r); - } + if let Some(r) = self.remote.lock().clone() + && r.revision == manifest.revision + { + return Ok(r); } self.open_remote(&manifest).await } @@ -787,7 +790,7 @@ impl RepoHandle { /// by the caller. Packs are never touched here (see `sync_packs_phase`). async fn sync_locked_inner(&self, span: &tracing::Span) -> Result<(), WalError> { let known = self.manifest_version.lock().clone(); - let outcome = crate::sync::freshness_check(&self.store, &known).await?; + let outcome = crate::sync::freshness_check(&self.store, known.as_ref()).await?; match outcome { crate::sync::SyncOutcome::Unchanged => self.update_freshness(), crate::sync::SyncOutcome::Changed { @@ -819,7 +822,7 @@ impl RepoHandle { let before = self.state.lock().applied_seq; crate::sync::apply_delta(self, &manifest, &meta_version).await?; span.record("entries_applied", manifest.head_seq.saturating_sub(before)); - *self.manifest.write() = Arc::new(manifest); + *self.manifest.write() = manifest; *self.manifest_version.lock() = Some(meta_version); self.update_freshness(); } @@ -854,16 +857,13 @@ impl RepoHandle { /// Any local pack that is a symlink into the store mount. fn has_linked_packs(&self) -> bool { - self.local - .packs() - .map(|ps| { - ps.iter() - .any(|p| self.local.pack_path(&p.checksum).is_symlink()) - }) - .unwrap_or(false) + self.local.packs().is_ok_and(|ps| { + ps.iter() + .any(|p| self.local.pack_path(&p.checksum).is_symlink()) + }) } - /// Internal serving sync (no read guard). Used by publish/checkpoint/read_log. + /// Internal serving sync (no read guard). Used by `publish/checkpoint/read_log`. pub(crate) async fn sync_impl(&self) -> Result<(), WalError> { self.sync_impl_level(SyncLevel::Serve).await } @@ -899,14 +899,11 @@ impl RepoHandle { .await; // Read manifest fresh - let (meta, manifest) = match crate::store_proto::get_message::( - &self.store, - walgit_proto::keys::MANIFEST, - ) - .await? - { - Some((m, manifest)) => (m, manifest), - None => return Err(WalError::NotFound), + let Some((meta, manifest)) = + crate::store_proto::get_message::(&self.store, walgit_proto::keys::MANIFEST) + .await? + else { + return Err(WalError::NotFound); }; // Reset state and re-materialize @@ -986,6 +983,7 @@ impl RepoHandle { synced: bool, created_at: Option, ) -> Result { + let sender = self.get_or_init_publisher()?; self.publish_waiters.fetch_add(1, Ordering::Relaxed); let (tx, rx) = tokio::sync::oneshot::channel(); let request = PublishRequest { @@ -997,7 +995,6 @@ impl RepoHandle { response: tx, }; - let sender = self.get_or_init_publisher().await; if sender.send(request).is_err() { self.publish_waiters.fetch_sub(1, Ordering::Relaxed); return Err(WalError::Corrupt("publisher channel closed".into())); @@ -1039,16 +1036,16 @@ impl RepoHandle { /// (never a failure on a read path). pub fn effective_config(&self) -> Arc { let settings = self.settings(); - let rev = settings.as_ref().map(|s| s.revision).unwrap_or(0); + let rev = settings.as_ref().map_or(0, |s| s.revision); if rev == 0 { return self.cfg.clone(); } - if let Some((r, c)) = self.effective.lock().as_ref() { - if *r == rev { - return c.clone(); - } + if let Some((r, c)) = self.effective.lock().as_ref() + && *r == rev + { + return c.clone(); } - let toml = settings.as_ref().map(|s| s.toml.as_str()).unwrap_or(""); + let toml = settings.as_ref().map_or("", |s| s.toml.as_str()); let cfg = match self.cfg.with_settings(toml) { Ok(c) => Arc::new(c), Err(e) => { @@ -1223,6 +1220,10 @@ impl RepoHandle { /// Read the checkpoint object's times when the manifest ref has none /// (one 240-byte GET per checkpoint per process; no-op otherwise). pub(crate) async fn learn_checkpoint_times(&self) -> Result<(), WalError> { + use walgit_store::ObjectStoreExt; + + use prost::Message; + let m = self.manifest(); let Some(cp) = m.checkpoint.as_ref() else { return Ok(()); @@ -1232,8 +1233,7 @@ impl RepoHandle { { return Ok(()); } - use prost::Message; - use walgit_store::ObjectStoreExt; + if let Some((_, bytes)) = self.store.get_bytes(&cp.key).await? { let cpo = walgit_proto::v1::Checkpoint::decode(bytes.as_ref()) .map_err(|e| WalError::Corrupt(format!("checkpoint decode: {e}")))?; @@ -1261,7 +1261,7 @@ impl RepoHandle { crate::log_reader::refs_at_seq(self, seq).await } - /// Read log entries [from_seq, to_seq]. + /// Read log entries [`from_seq`, `to_seq`]. pub async fn read_log( &self, from_seq: u64, @@ -1288,14 +1288,14 @@ impl RepoHandle { *self.last_freshness.lock() = Some(Instant::now()); } - async fn get_or_init_publisher(&self) -> mpsc::UnboundedSender { + fn get_or_init_publisher(&self) -> Result, WalError> { let mut guard = self.publish_tx.lock(); if let Some(tx) = &*guard { // A publisher task that died (panic mid-batch) leaves a sender to // a dropped receiver; respawn instead of failing every push on // this instance forever. if !tx.is_closed() { - return tx.clone(); + return Ok(tx.clone()); } tracing::warn!(repo = %self.id, "publisher task is gone; respawning"); } @@ -1303,10 +1303,12 @@ impl RepoHandle { let arc = self .self_arc .get() - .expect("self_arc must be set before publish") + .ok_or_else(|| { + WalError::Corrupt("publisher repository reference not initialized".into()) + })? .clone(); tokio::spawn(crate::publish::publisher_task(arc, rx)); *guard = Some(tx.clone()); - tx + Ok(tx) } } diff --git a/crates/walgit-wal/src/lockwait.rs b/crates/walgit-wal/src/lockwait.rs index 6a0e6d2..5520d74 100644 --- a/crates/walgit-wal/src/lockwait.rs +++ b/crates/walgit-wal/src/lockwait.rs @@ -26,7 +26,7 @@ pub fn record( warn_after: Duration, ) { metrics::histogram!("walgit_lock_wait_seconds", "lock" => lock).record(waited.as_secs_f64()); - let ms = waited.as_millis() as u64; + let ms = u64::try_from(waited.as_millis()).unwrap_or(u64::MAX); { let mut s = STATS.lock(); match s.iter_mut().find(|(l, _, _)| *l == lock) { diff --git a/crates/walgit-wal/src/log_reader.rs b/crates/walgit-wal/src/log_reader.rs index 78f11bc..3b23594 100644 --- a/crates/walgit-wal/src/log_reader.rs +++ b/crates/walgit-wal/src/log_reader.rs @@ -6,7 +6,7 @@ use walgit_store::{GetOptions, GetResult, ObjectStore}; use crate::error::WalError; use crate::handle::RepoHandle; -/// Read log entries in [from_seq, to_seq]. If `to_seq` is None, read up to +/// Read log entries in [`from_seq`, `to_seq`]. If `to_seq` is None, read up to /// `manifest.head_seq`. pub(crate) async fn read_log_impl( handle: &RepoHandle, @@ -18,9 +18,9 @@ pub(crate) async fn read_log_impl( // the repo's write lock here would deadlock callers that hold a read // guard (overview, tests), and freshness_ttl=0 makes that the common case. let known = handle.manifest_version.lock().clone(); - let manifest = match crate::sync::freshness_check(&handle.store, &known).await? { + let manifest = match crate::sync::freshness_check(&handle.store, known.as_ref()).await? { crate::sync::SyncOutcome::Unchanged => handle.manifest.read().clone(), - crate::sync::SyncOutcome::Changed { manifest, .. } => std::sync::Arc::new(manifest), + crate::sync::SyncOutcome::Changed { manifest, .. } => manifest, }; let head_seq = manifest.head_seq; let to = to_seq.unwrap_or(head_seq).min(head_seq); @@ -41,7 +41,11 @@ pub(crate) async fn read_log_impl( let res = handle.store.get(&seg.key, GetOptions::default()).await?; let bytes = match res { GetResult::Object { meta, body } => { - walgit_store::util::collect(body, meta.size as usize).await? + walgit_store::util::collect( + body, + usize::try_from(meta.size).map_err(|e| WalError::Corrupt(e.to_string()))?, + ) + .await? } GetResult::NotModified { .. } => continue, }; @@ -118,7 +122,7 @@ async fn replay_refs( handle.learn_checkpoint_times().await?; let times = handle.checkpoint_times(); let cp_time = times.and_then(|t| t.as_of.or(t.created_at)); - cp_time.map(|t| t <= at).unwrap_or(false) + cp_time.is_some_and(|t| t <= at) } }; if usable { @@ -151,7 +155,7 @@ async fn replay_refs( match cut { Cut::Time(at) => { let t = e.created_at.as_ref().map(walgit_proto::time::to_system); - if t.map(|t| t > at).unwrap_or(false) { + if t.is_some_and(|t| t > at) { break; } } @@ -165,7 +169,7 @@ async fn replay_refs( for u in &txn.updates { if !u.new_symbolic_target.is_empty() { if u.name == "HEAD" { - head_target = u.new_symbolic_target.clone(); + head_target.clone_from(&u.new_symbolic_target); } continue; } diff --git a/crates/walgit-wal/src/progress.rs b/crates/walgit-wal/src/progress.rs index 8750db4..e9e6952 100644 --- a/crates/walgit-wal/src/progress.rs +++ b/crates/walgit-wal/src/progress.rs @@ -44,9 +44,10 @@ impl Progress { total: Option, unit: &'static str, ) -> Self { - let percent = total - .filter(|t| *t > 0) - .map(|t| ((done as f64 / t as f64) * 1000.0).round() / 10.0); + let percent = total.filter(|t| *t > 0).map(|t| { + let tenths = done.min(t).saturating_mul(1_000) / t; + f64::from(u32::try_from(tenths).unwrap_or(1_000)) / 10.0 + }); Progress::Progress { label: label.into(), done, @@ -140,7 +141,10 @@ impl Throttle { } /// True when an update should be emitted now. pub fn tick(&self, force: bool) -> bool { - let mut last = self.last.lock().unwrap(); + let mut last = self + .last + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let now = std::time::Instant::now(); if force || last diff --git a/crates/walgit-wal/src/publish.rs b/crates/walgit-wal/src/publish.rs index 5aeac64..6d92f99 100644 --- a/crates/walgit-wal/src/publish.rs +++ b/crates/walgit-wal/src/publish.rs @@ -1,7 +1,8 @@ //! Publish path: linearizable CAS with batching. +#![allow(clippy::needless_continue)] //! //! Design: -//! Each RepoHandle has a single-flight publisher task. `publish_push` and +//! Each `RepoHandle` has a single-flight publisher task. `publish_push` and //! `publish_ref_update` enqueue a [`PublishRequest`] onto an mpsc channel //! and await a oneshot response. The publisher collects requests within //! `cfg.wal.batch_window` (up to `max_batch`), then processes them as one @@ -62,7 +63,7 @@ pub(crate) struct PublishRequest { /// True when receive-pack already performed the request freshness check. pub(crate) synced: bool, /// Explicit entry time (history replay); None = now. Validated monotonic - /// (>= the head entry's created_at) before the batch is written. + /// (>= the head entry's `created_at`) before the batch is written. pub(crate) created_at: Option, pub(crate) response: oneshot::Sender>, } @@ -138,10 +139,7 @@ pub(crate) async fn put_immutable_create( // Big packs go up striped (parts + server-side compose, ~8 × 100 MB/s): // a large repository's rebuilt base (32.4 GB) took 431 s single-stream at 75 MB/s in the // weekly dry run of 2026-08-21. Small packs (every push) stay one PUT. - let size = tokio::fs::metadata(&path) - .await - .map(|m| m.len()) - .unwrap_or(0); + let size = tokio::fs::metadata(&path).await.map_or(0, |m| m.len()); let put = if size >= PARALLEL_PUT_MIN_BYTES && store.supports_compose() { walgit_store::util::put_file_parallel(store, &key, &path, opts(), PARALLEL_PUT_STRIPES) .await @@ -160,27 +158,26 @@ pub(crate) async fn put_immutable_create( // hiccup) must not leave a referenced object missing. Rare path, // one HEAD; on a miss, write it unconditionally (content-addressed: // whoever wins wrote the same bytes). - match store.head(&key).await? { - Some(_) => Ok(()), - None => { - tracing::warn!( - key, - "create-if-absent reported the object present but HEAD finds nothing; writing it" - ); - store - .put( - &key, - PutBody::File(path), - PutOptions { - mode: PutMode::Overwrite, - immutable: true, - ..Default::default() - }, - ) - .await - .map(|_| ()) - .map_err(WalError::Store) - } + if store.head(&key).await?.is_some() { + Ok(()) + } else { + tracing::warn!( + key, + "create-if-absent reported the object present but HEAD finds nothing; writing it" + ); + store + .put( + &key, + PutBody::File(path), + PutOptions { + mode: PutMode::Overwrite, + immutable: true, + ..Default::default() + }, + ) + .await + .map(|_| ()) + .map_err(WalError::Store) } } Err(e) => Err(WalError::Store(e)), @@ -213,7 +210,7 @@ const ORPHAN_GRACE_PROBES: u32 = 3; const ORPHAN_GRACE_STEP: std::time::Duration = std::time::Duration::from_millis(100); /// Never burn more than this many seqs in one claim (a pile of orphans means /// something else is wrong). -const MAX_BURN: u32 = 8; +const MAX_BURN: usize = 8; /// Unconditional fresh read of the manifest (not the handle's cached view). pub(crate) async fn read_manifest_fresh(store: &Prefixed) -> Result, WalError> { @@ -259,7 +256,7 @@ pub(crate) async fn claim_log_slot( let mut probes = 0u32; let orphan_version = loop { let fresh = read_manifest_fresh(store).await?; - let fresh_head = fresh.as_ref().map(|m| m.head_seq).unwrap_or(0); + let fresh_head = fresh.as_ref().map_or(0, |m| m.head_seq); if fresh_head >= seq { return Ok(ClaimOutcome::Contended); } @@ -285,7 +282,7 @@ pub(crate) async fn claim_log_slot( "orphaned log segment at the head (writer crashed between log PUT and manifest CAS); burning the seq" ); burned.push((key, v)); - if burned.len() as u32 >= MAX_BURN { + if burned.len() >= MAX_BURN { return Err(WalError::Corrupt(format!( "{MAX_BURN} consecutive orphaned log segments from seq {}", head_seq + 1 @@ -416,10 +413,7 @@ pub(crate) async fn publisher_task( let max_batch = handle.cfg.wal.max_batch; loop { - let first = match rx.recv().await { - Some(r) => r, - None => break, - }; + let Some(first) = rx.recv().await else { break }; let mut batch = Vec::with_capacity(max_batch.min(64)); batch.push(first); @@ -439,7 +433,7 @@ pub(crate) async fn publisher_task( tokio::pin!(deadline); loop { tokio::select! { - _ = &mut deadline => break, + () = &mut deadline => break, maybe_req = rx.recv() => { match maybe_req { Some(r) => { @@ -515,21 +509,21 @@ async fn process_batch(handle: &RepoHandle, batch: Vec) -> Resul // must be monotonic: >= the head entry's time, >= earlier explicit // times in this batch — the WAL's created_at order is history). let mut verified: Vec = Vec::with_capacity(batch.len()); - let mut floor: Option = handle.last_entry_time.lock().clone(); + let mut floor: Option = *handle.last_entry_time.lock(); for req in &batch { let mut per_ref = verify_txn(&req.txn, &working_refs); if let Some(ts) = &req.created_at { let t = time::to_system(ts); - if let Some(f) = floor { - if t < f { - let msg = format!( - "created_at {} is before the WAL head's {} (entries must be monotonic)", - chrono::DateTime::::from(t).to_rfc3339(), - chrono::DateTime::::from(f).to_rfc3339() - ); - for (_, r) in per_ref.iter_mut() { - *r = Err(RefError::Rejected(msg.clone())); - } + if let Some(f) = floor + && t < f + { + let msg = format!( + "created_at {} is before the WAL head's {} (entries must be monotonic)", + chrono::DateTime::::from(t).to_rfc3339(), + chrono::DateTime::::from(f).to_rfc3339() + ); + for (_, r) in &mut per_ref { + *r = Err(RefError::Rejected(msg.clone())); } } if per_ref.iter().all(|(_, r)| r.is_ok()) { @@ -575,9 +569,13 @@ async fn process_batch(handle: &RepoHandle, batch: Vec) -> Resul let build = |first_seq: u64| -> (Vec, Vec) { let mut entries = Vec::with_capacity(valid_indices.len()); let mut new_packs = Vec::new(); - for (offset, &idx) in valid_indices.iter().enumerate() { + for (offset, (req, _)) in batch + .iter() + .zip(&verified) + .filter(|(_, v)| v.valid) + .enumerate() + { let seq = first_seq + offset as u64; - let req = &batch[idx]; let pack_ref = req.pack.as_ref().map(|p| pack_ref_from_ingested(p, seq)); if let Some(pr) = &pack_ref { new_packs.push(pr.clone()); @@ -590,7 +588,7 @@ async fn process_batch(handle: &RepoHandle, batch: Vec) -> Resul Vec::new(), &req.meta, &writer, - req.created_at.clone(), + req.created_at, )); } (entries, new_packs) @@ -636,12 +634,18 @@ async fn process_batch(handle: &RepoHandle, batch: Vec) -> Resul } Err(e) => { let msg = e.to_string(); - return finish_with_error_msg(batch, &valid_indices, msg, e); + return finish_with_error_msg(batch, &valid_indices, &msg, e); } }; let first_seq = slot.first_seq; let (entries, new_packs) = build(first_seq); - let last_seq = entries.last().unwrap().seq; + let Some(last_seq) = entries.last().map(|e| e.seq) else { + return finish_with_error( + batch, + &valid_indices, + WalError::Corrupt("empty publish log batch".into()), + ); + }; // 6. Build updated manifest let mut updated: Manifest = (*manifest).clone(); @@ -659,7 +663,7 @@ async fn process_batch(handle: &RepoHandle, batch: Vec) -> Resul updated.log_segments.push(seg_ref); updated.log_segments.sort_by_key(|s| s.first_seq); updated.updated_at = Some(time::now()); - updated.writer = writer.to_string(); + updated.writer = writer.clone(); updated.revision += 1; // CAS manifest @@ -696,165 +700,160 @@ async fn process_batch(handle: &RepoHandle, batch: Vec) -> Resul // it and sweeps it; deleting here could race a lost-response // commit that `cas_landed` itself failed to observe. let msg = e.to_string(); - return finish_with_error_msg(batch, &valid_indices, msg, WalError::Store(e)); + return finish_with_error_msg(batch, &valid_indices, &msg, WalError::Store(e)); } Err(e2) => { let msg = format!("{e} (and re-reading the manifest failed: {e2})"); - return finish_with_error_msg(batch, &valid_indices, msg, WalError::Store(e)); + return finish_with_error_msg(batch, &valid_indices, &msg, WalError::Store(e)); } }, }; - match committed { - Some((committed, version)) => { - // Success! Update handle state. A landed-but-errored CAS leaves - // us without the new version: drop our cached one so the next - // sync refetches unconditionally. - let version = match version { - Some(v) => v, - None => match handle.store.head(keys::MANIFEST).await? { - Some(m) => m.version, - None => { - return finish_with_error( - batch, - &valid_indices, - WalError::Corrupt("manifest vanished after commit".into()), - ); - } - }, - }; - // The local commit — ref txns applied, then the new manifest version advertised — happens - // under `sync_mutex`, the lock the refs phase of every sync holds: a sync that already read - // the committed manifest would otherwise replay the same entry concurrently (two - // `git update-ref` on one ref → a lock collision: rig round 2447 of 2450, 2026-08-23) and a - // reader between the two steps would see one without the other. Refs first: the - // advertisement/ls-refs caches are keyed by the manifest version, and the reverse order let - // a reader cache the OLD refs under the NEW version (1 round in 6 on the rig). - // - // The WAL commit already happened (CAS ok) and is the truth: whatever the local apply does, - // every waiter is answered `ok`. A failed apply leaves the version unadvertised, so the next - // sync sees a change and replays the entry — the copy repairs itself. (Answering an error - // here produced a durable push that git reported as failed — "0 winners", commit fetchable.) - let mut local_ok = true; - { - let _sync_guard = crate::lockwait::timed( - "sync_mutex", - &handle.id, - handle.cfg.telemetry.lock_wait_warn, - || handle.sync_mutex.try_lock().ok(), - handle.sync_mutex.lock(), - ) - .await; - for &idx in &valid_indices { - if let Err(e) = handle.local.apply_ref_txn(&batch[idx].txn, false) { - tracing::warn!(repo = %handle.id, seq = last_seq, error = %e, "published (CAS ok), but applying the ref txn to the local copy failed; the next sync replays it"); - metrics::counter!("walgit_publish_local_apply_failed_total") - .increment(1); - local_ok = false; - break; - } + if let Some((committed, version)) = committed { + // Success! Update handle state. A landed-but-errored CAS leaves + // us without the new version: drop our cached one so the next + // sync refetches unconditionally. + let version = match version { + Some(v) => v, + None => match handle.store.head(keys::MANIFEST).await? { + Some(m) => m.version, + None => { + return finish_with_error( + batch, + &valid_indices, + WalError::Corrupt("manifest vanished after commit".into()), + ); } - if local_ok && let Err(e) = handle.local.refresh_async().await { - tracing::warn!(repo = %handle.id, seq = last_seq, error = %e, "published (CAS ok), but refreshing the local copy failed; the next sync repairs it"); + }, + }; + // The local commit — ref txns applied, then the new manifest version advertised — happens + // under `sync_mutex`, the lock the refs phase of every sync holds: a sync that already read + // the committed manifest would otherwise replay the same entry concurrently (two + // `git update-ref` on one ref → a lock collision: rig round 2447 of 2450, 2026-08-23) and a + // reader between the two steps would see one without the other. Refs first: the + // advertisement/ls-refs caches are keyed by the manifest version, and the reverse order let + // a reader cache the OLD refs under the NEW version (1 round in 6 on the rig). + // + // The WAL commit already happened (CAS ok) and is the truth: whatever the local apply does, + // every waiter is answered `ok`. A failed apply leaves the version unadvertised, so the next + // sync sees a change and replays the entry — the copy repairs itself. (Answering an error + // here produced a durable push that git reported as failed — "0 winners", commit fetchable.) + let mut local_ok = true; + { + let _sync_guard = crate::lockwait::timed( + "sync_mutex", + &handle.id, + handle.cfg.telemetry.lock_wait_warn, + || handle.sync_mutex.try_lock().ok(), + handle.sync_mutex.lock(), + ) + .await; + for (req, _) in batch.iter().zip(&verified).filter(|(_, v)| v.valid) { + if let Err(e) = handle.local.apply_ref_txn(&req.txn, false) { + tracing::warn!(repo = %handle.id, seq = last_seq, error = %e, "published (CAS ok), but applying the ref txn to the local copy failed; the next sync replays it"); + metrics::counter!("walgit_publish_local_apply_failed_total").increment(1); local_ok = false; + break; } - // Test hook: widen the gap between the two local-commit steps (harmless in this order - // and under this lock; the poison window with the steps reversed and no lock). - if let Some(ms) = std::env::var("WALGIT_TEST_PUBLISH_GAP_MS") - .ok() - .and_then(|v| v.parse::().ok()) + } + if local_ok && let Err(e) = handle.local.refresh_async().await { + tracing::warn!(repo = %handle.id, seq = last_seq, error = %e, "published (CAS ok), but refreshing the local copy failed; the next sync repairs it"); + local_ok = false; + } + // Test hook: widen the gap between the two local-commit steps (harmless in this order + // and under this lock; the poison window with the steps reversed and no lock). + if let Some(ms) = std::env::var("WALGIT_TEST_PUBLISH_GAP_MS") + .ok() + .and_then(|v| v.parse::().ok()) + { + tokio::time::sleep(std::time::Duration::from_millis(ms)).await; + } + if local_ok { + *handle.manifest.write() = Arc::new(committed.clone()); + *handle.manifest_version.lock() = Some(version.clone()); { - tokio::time::sleep(std::time::Duration::from_millis(ms)).await; - } - if local_ok { - *handle.manifest.write() = Arc::new(committed.clone()); - *handle.manifest_version.lock() = Some(version.clone()); - { - let mut state = handle.state.lock(); - state.manifest_version = Some(version.as_str().to_string()); - state.applied_seq = last_seq; - let ready = state.packs_ready(); - state.revision = committed.revision; - if ready { - state.packs_revision = committed.revision; - } - } - if let Err(e) = crate::state::save_state( - handle.local.path(), - &handle.state.lock().clone(), - ) { - tracing::warn!(repo = %handle.id, error = %e, "published (CAS ok), but saving local state failed; the next sync repairs it"); + let mut state = handle.state.lock(); + state.manifest_version = Some(version.as_str().to_string()); + state.applied_seq = last_seq; + let ready = state.packs_ready(); + state.revision = committed.revision; + if ready { + state.packs_revision = committed.revision; } - } else { - // Forget the known version so the next sync performs an unconditional GET and - // replays from the last applied seq. - handle.manifest_version.lock().take(); } - } - sweep_burned(&handle.store, &slot) - .instrument(span.clone()) - .await; - - for e in &entries { - if let Some(t) = e.created_at.as_ref() { - note_entry_time(handle, e.seq, t); - } - } - // Fold the pushed packs' commits into the local commit-graph - // chain (cheap, incremental; off the client's critical path). - if !new_packs.is_empty() { - if let Some(arc) = handle.self_arc.get().cloned() { - let packs = new_packs.clone(); - tokio::spawn(async move { - let manifest = arc.manifest(); - crate::sync::maintain_commit_graph(&arc, &manifest, &packs).await; - }); + if let Err(e) = + crate::state::save_state(handle.local.path(), &handle.state.lock().clone()) + { + tracing::warn!(repo = %handle.id, error = %e, "published (CAS ok), but saving local state failed; the next sync repairs it"); } + } else { + // Forget the known version so the next sync performs an unconditional GET and + // replays from the last applied seq. + handle.manifest_version.lock().take(); } + } + sweep_burned(&handle.store, &slot) + .instrument(span.clone()) + .await; - // Build all responses (success for valid, rejection for invalid) - let mut responses: Vec = Vec::with_capacity(batch.len()); - for (i, v) in verified.iter().enumerate() { - if v.valid { - let offset = valid_indices.iter().position(|&vi| vi == i).unwrap(); - let seq = first_seq + offset as u64; - responses.push(PublishResult { - seq, - per_ref: v.per_ref.clone(), - }); - } else { - responses.push(PublishResult { - seq: 0, - per_ref: v.per_ref.clone(), - }); - } + for e in &entries { + if let Some(t) = e.created_at.as_ref() { + note_entry_time(handle, e.seq, t); } + } + // Fold the pushed packs' commits into the local commit-graph + // chain (cheap, incremental; off the client's critical path). + if !new_packs.is_empty() + && let Some(arc) = handle.self_arc.get().cloned() + { + let packs = new_packs.clone(); + tokio::spawn(async move { + let manifest = arc.manifest(); + crate::sync::maintain_commit_graph(&arc, &manifest, &packs).await; + }); + } - // Consume batch and send responses - for (req, resp) in batch.into_iter().zip(responses) { - let _ = req.response.send(Ok(resp)); + // Build all responses (success for valid, rejection for invalid) + let mut responses: Vec = Vec::with_capacity(batch.len()); + let mut valid_offset = 0u64; + for v in &verified { + if v.valid { + let seq = first_seq + valid_offset; + valid_offset += 1; + responses.push(PublishResult { + seq, + per_ref: v.per_ref.clone(), + }); + } else { + responses.push(PublishResult { + seq: 0, + per_ref: v.per_ref.clone(), + }); } - - // Maybe trigger checkpoint - maybe_trigger_checkpoint(handle, last_seq); - - span.record("seq", last_seq); - span.record("cas_retries", attempts); - return Ok(()); } - None => { - // Lost the CAS: drop exactly the segment we wrote, re-sync, retry. - drop_own_slot(&handle.store, &slot) - .instrument(span.clone()) - .await; - attempts += 1; - if attempts >= max_retries { - span.record("cas_retries", attempts); - return finish_with_error(batch, &valid_indices, WalError::Retry { attempts }); - } - continue; + + // Consume batch and send responses + for (req, resp) in batch.into_iter().zip(responses) { + let _ = req.response.send(Ok(resp)); } + + // Maybe trigger checkpoint + maybe_trigger_checkpoint(handle, last_seq); + + span.record("seq", last_seq); + span.record("cas_retries", attempts); + return Ok(()); } + // Lost the CAS: drop exactly the segment we wrote, re-sync, retry. + drop_own_slot(&handle.store, &slot) + .instrument(span.clone()) + .await; + attempts += 1; + if attempts >= max_retries { + span.record("cas_retries", attempts); + return finish_with_error(batch, &valid_indices, WalError::Retry { attempts }); + } + continue; } } @@ -868,7 +867,7 @@ fn finish_all_errors(batch: Vec, err: WalError) -> Result<(), Wa Err(err) } /// Send error responses to valid request senders, then return the error. -/// Converts the error to a string for each sender since WalError is not Clone. +/// Converts the error to a string for each sender since `WalError` is not Clone. fn finish_with_error( batch: Vec, valid_indices: &[usize], @@ -880,7 +879,7 @@ fn finish_with_error( // batch error too rather than dropping the channel ("publisher dropped // response" told the caller nothing). let _ = valid_indices; - for req in batch.into_iter() { + for req in batch { let _ = req.response.send(Err(WalError::Corrupt(msg.clone()))); } Err(err) @@ -889,12 +888,12 @@ fn finish_with_error( fn finish_with_error_msg( batch: Vec, valid_indices: &[usize], - msg: String, + msg: &str, err: WalError, ) -> Result<(), WalError> { let _ = valid_indices; - for req in batch.into_iter() { - let _ = req.response.send(Err(WalError::Corrupt(msg.clone()))); + for req in batch { + let _ = req.response.send(Err(WalError::Corrupt(msg.to_owned()))); } Err(err) } @@ -903,19 +902,19 @@ fn maybe_trigger_checkpoint(handle: &RepoHandle, _head_seq: u64) { // Opportunistic: the writer that crossed a trigger folds the log. The // `maintain` role covers repos nobody pushes to (age trigger). let due = crate::checkpoint::checkpoint_due(&handle.manifest.read(), &handle.cfg.wal); - if let Some(trigger) = due { - if let Some(arc) = handle.self_arc.get().cloned() { - tokio::spawn(async move { - match crate::checkpoint::write_checkpoint_impl(&arc).await { - Ok(cp) => { - tracing::info!(repo = %arc.id, seq = cp.seq, %trigger, "auto checkpoint written") - } - Err(e) => { - tracing::warn!(repo = %arc.id, %trigger, "auto checkpoint failed: {e}") - } + if let Some(trigger) = due + && let Some(arc) = handle.self_arc.get().cloned() + { + tokio::spawn(async move { + match crate::checkpoint::write_checkpoint_impl(&arc).await { + Ok(cp) => { + tracing::info!(repo = %arc.id, seq = cp.seq, %trigger, "auto checkpoint written"); } - }); - } + Err(e) => { + tracing::warn!(repo = %arc.id, %trigger, "auto checkpoint failed: {e}"); + } + } + }); } } @@ -984,7 +983,10 @@ pub(crate) async fn publish_compact_impl( } let pack_ref = pack_ref_from_info(&new_pack, 0, tier); // seq set below - let supersedes_hex: Vec = supersedes.iter().map(|o| o.to_string()).collect(); + let supersedes_hex: Vec = supersedes + .iter() + .map(std::string::ToString::to_string) + .collect(); let mut attempts = 0u32; @@ -1008,7 +1010,7 @@ pub(crate) async fn publish_compact_impl( supersedes: supersedes_hex.clone(), checkpoint: None, created_at: Some(entry_time), - writer: writer.to_string(), + writer: writer.clone(), meta: HashMap::new(), settings: None, }; @@ -1034,8 +1036,10 @@ pub(crate) async fn publish_compact_impl( // Build updated manifest let mut updated: Manifest = (*manifest).clone(); updated.head_seq = seq; - let sup_set: std::collections::HashSet<&str> = - supersedes_hex.iter().map(|s| s.as_str()).collect(); + let sup_set: std::collections::HashSet<&str> = supersedes_hex + .iter() + .map(std::string::String::as_str) + .collect(); updated .packs .retain(|p| !sup_set.contains(p.checksum.as_str()) && p.checksum != pack_ref.checksum); @@ -1055,7 +1059,7 @@ pub(crate) async fn publish_compact_impl( updated.log_segments.push(seg_ref); updated.log_segments.sort_by_key(|s| s.first_seq); updated.updated_at = Some(time::now()); - updated.writer = writer.to_string(); + updated.writer = writer.clone(); updated.revision += 1; let buf = updated.encode_to_vec(); @@ -1088,46 +1092,41 @@ pub(crate) async fn publish_compact_impl( })?; Some((fresh, v)) } - Ok(None) => return Err(WalError::Store(e)), - Err(_) => return Err(WalError::Store(e)), + Ok(None) | Err(_) => return Err(WalError::Store(e)), }, }; - match committed { - Some((committed, version)) => { - *handle.manifest.write() = Arc::new(committed.clone()); - *handle.manifest_version.lock() = Some(version.clone()); - note_entry_time(handle, seq, &entry_time); - { - let mut state = handle.state.lock(); - state.manifest_version = Some(version.as_str().to_string()); - state.applied_seq = seq; - // The publisher's own superseded packs are removed by the next pack sync like - // everyone else's (a scratch-copy base rebuild leaves them in the serving copy; - // a geometric fold already deleted them — the removal is then a no-op). - for s in &supersedes_hex { - if !state.pending_pack_removals.contains(s) { - state.pending_pack_removals.push(s.clone()); - } - } - let ready = state.packs_ready(); - state.revision = committed.revision; - if ready { - state.packs_revision = committed.revision; + if let Some((committed, version)) = committed { + *handle.manifest.write() = Arc::new(committed.clone()); + *handle.manifest_version.lock() = Some(version.clone()); + note_entry_time(handle, seq, &entry_time); + { + let mut state = handle.state.lock(); + state.manifest_version = Some(version.as_str().to_string()); + state.applied_seq = seq; + // The publisher's own superseded packs are removed by the next pack sync like + // everyone else's (a scratch-copy base rebuild leaves them in the serving copy; + // a geometric fold already deleted them — the removal is then a no-op). + for s in &supersedes_hex { + if !state.pending_pack_removals.contains(s) { + state.pending_pack_removals.push(s.clone()); } } - crate::state::save_state(handle.local.path(), &handle.state.lock().clone())?; - sweep_burned(&handle.store, &slot).await; - return Ok(seq); - } - None => { - drop_own_slot(&handle.store, &slot).await; - attempts += 1; - if attempts >= max_retries { - return Err(WalError::Retry { attempts }); + let ready = state.packs_ready(); + state.revision = committed.revision; + if ready { + state.packs_revision = committed.revision; } - continue; } + crate::state::save_state(handle.local.path(), &handle.state.lock().clone())?; + sweep_burned(&handle.store, &slot).await; + return Ok(seq); + } + drop_own_slot(&handle.store, &slot).await; + attempts += 1; + if attempts >= max_retries { + return Err(WalError::Retry { attempts }); } + continue; } } @@ -1202,7 +1201,7 @@ pub(crate) async fn annotate_pack_impl( } let pack_ref = p.clone(); updated.updated_at = Some(time::now()); - updated.writer = writer.to_string(); + updated.writer = writer.clone(); updated.revision += 1; let mode = match &known_version { Some(v) => PutMode::Update(v.clone()), @@ -1263,7 +1262,10 @@ pub(crate) async fn add_pack_impl( let checksum = gix_hash::ObjectId::from_hex(hex.as_bytes()) .map_err(|e| WalError::Corrupt(format!("bad pack name {name}: {e}")))?; let dest = handle.local.pack_path(&checksum); - std::fs::create_dir_all(dest.parent().unwrap())?; + std::fs::create_dir_all( + dest.parent() + .ok_or_else(|| WalError::Corrupt("destination has no parent".into()))?, + )?; for (src, dst) in [(pack, dest.clone()), (idx, dest.with_extension("idx"))] { if !dst.exists() && std::fs::hard_link(src, &dst).is_err() { std::fs::copy(src, &dst)?; @@ -1299,7 +1301,7 @@ pub(crate) async fn publish_settings_impl( handle.sync_impl_level(crate::sync::SyncLevel::Refs).await?; let manifest = handle.manifest.read().clone(); let known_version = handle.manifest_version.lock().clone(); - let revision = manifest.settings.as_ref().map(|s| s.revision).unwrap_or(0) + 1; + let revision = manifest.settings.as_ref().map_or(0, |s| s.revision) + 1; let settings = walgit_proto::v1::RepoSettings { toml: toml_text.to_string(), revision, @@ -1316,7 +1318,7 @@ pub(crate) async fn publish_settings_impl( supersedes: Vec::new(), checkpoint: None, created_at: Some(entry_time), - writer: writer.to_string(), + writer: writer.clone(), meta: HashMap::from([ ("author".to_string(), author.to_string()), ("message".to_string(), message.to_string()), @@ -1353,7 +1355,7 @@ pub(crate) async fn publish_settings_impl( }); updated.log_segments.sort_by_key(|s| s.first_seq); updated.updated_at = Some(time::now()); - updated.writer = writer.to_string(); + updated.writer = writer.clone(); updated.revision += 1; let buf = updated.encode_to_vec(); let mode = match &known_version { diff --git a/crates/walgit-wal/src/registry.rs b/crates/walgit-wal/src/registry.rs index d248b9d..6f30f8f 100644 --- a/crates/walgit-wal/src/registry.rs +++ b/crates/walgit-wal/src/registry.rs @@ -1,4 +1,5 @@ -//! Registry: process-wide map of RepoId -> Arc. +//! Registry: process-wide map of `RepoId` -> Arc. +#![allow(clippy::unnecessary_wraps)] use std::str::FromStr; use std::sync::Arc; @@ -95,18 +96,17 @@ impl Registry { let prefixed = Prefixed::new(self.store.clone(), prefix); // Read manifest (NotFound if absent) - let (meta, manifest) = match get_message::(&prefixed, keys::MANIFEST).await? { - Some(v) => v, - None => return Err(WalError::NotFound), + let Some((meta, manifest)) = get_message::(&prefixed, keys::MANIFEST).await? + else { + return Err(WalError::NotFound); }; // Open or init local repo (LocalRepo joins owner/name.git onto the root). - let local = match LocalRepo::open(&self.cache_root, id)? { - Some(l) => l, - None => { - let format = parse_object_format(&manifest.object_format); - LocalRepo::init(&self.cache_root, id, format)? - } + let local = if let Some(l) = LocalRepo::open(&self.cache_root, id)? { + l + } else { + let format = parse_object_format(&manifest.object_format); + LocalRepo::init(&self.cache_root, id, format)? }; // Load state @@ -183,7 +183,7 @@ impl Registry { Ok(()) } - /// CAS-create manifest.pb (PutMode::Create). Err(AlreadyExists) on 412. + /// CAS-create manifest.pb (`PutMode::Create`). Err(AlreadyExists) on 412. pub async fn create( &self, id: &RepoId, @@ -341,8 +341,21 @@ impl Registry { Ok(repos) } - /// Disk cache maintenance: evict idle repos beyond cache.max_bytes / evict_idle_after. - pub async fn evict_idle(&self) -> Result { + /// Disk cache maintenance: evict idle repos beyond `cache.max_bytes` / `evict_idle_after`. + pub async fn evict_idle(self: &Arc) -> Result { + let registry = Arc::clone(self); + tokio::task::spawn_blocking(move || registry.evict_idle_blocking()) + .await + .map_err(|e| WalError::Corrupt(format!("cache eviction task: {e}")))? + } + + #[expect( + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "Disk watermarks are approximate nonnegative fractions, with truncation to whole bytes" + )] + fn evict_idle_blocking(&self) -> Result { let evict_after = self.cfg.cache.evict_idle_after; // D25: budget mode evicts past `cache.max_bytes`; disk mode only under // disk pressure (filesystem of `cache.dir` above `disk_high_watermark`) @@ -385,7 +398,7 @@ impl Registry { // Collect idle repos. In-use checks happen again while evicting: a // request may acquire a ReadGuard after this snapshot. - for entry in self.repos.iter() { + for entry in &self.repos { let handle = entry.value(); let last_access = handle.last_access(); if now.duration_since(last_access) > evict_after { @@ -469,8 +482,13 @@ fn disk_usage(path: &std::path::Path) -> Option<(u64, u64)> { use std::ffi::CString; use std::os::unix::ffi::OsStrExt; let c = CString::new(path.as_os_str().as_bytes()).ok()?; + // SAFETY: statvfs is a C integer struct; all-zero is a valid initialized value. + #[allow(unsafe_code)] let mut st: libc::statvfs = unsafe { std::mem::zeroed() }; - if unsafe { libc::statvfs(c.as_ptr(), &mut st) } != 0 { + // SAFETY: c is NUL-terminated and live; st is aligned writable storage for statvfs. + #[allow(unsafe_code)] + let result = unsafe { libc::statvfs(c.as_ptr(), &raw mut st) }; + if result != 0 { return None; } let total = st.f_blocks as u64 * st.f_frsize as u64; diff --git a/crates/walgit-wal/src/remote.rs b/crates/walgit-wal/src/remote.rs index 303ed7c..8ea697f 100644 --- a/crates/walgit-wal/src/remote.rs +++ b/crates/walgit-wal/src/remote.rs @@ -40,7 +40,7 @@ impl BlockCache { cache: moka::future::Cache::builder() .max_capacity(max_bytes.max(BLOCK_SIZE * 4)) .weigher(|_k: &(Arc, u64), v: &Bytes| { - v.len().clamp(1, u32::MAX as usize) as u32 + u32::try_from(v.len().max(1)).unwrap_or(u32::MAX) }) .build(), range_reads: AtomicU64::new(0), @@ -90,7 +90,7 @@ impl BlockCache { return Err(WalError::Corrupt(format!("unexpected 304 for {key}"))); } }; - let b = walgit_store::util::collect(body, (end - start) as usize).await?; + let b = walgit_store::util::collect(body, usize::try_from(end - start).map_err(|e| WalError::Corrupt(e.to_string()))?).await?; if b.len() as u64 != end - start { return Err(WalError::Corrupt(format!( "short range read for {key}: {start}..{end} got {}", @@ -175,10 +175,10 @@ impl RemotePacks { if let Ok(mut rd) = tokio::fs::read_dir(&dir).await { while let Ok(Some(e)) = rd.next_entry().await { let name = e.file_name().to_string_lossy().to_string(); - if let Some(stem) = name.strip_suffix(".idx") { - if !live.contains(stem) { - let _ = tokio::fs::remove_file(e.path()).await; - } + if let Some(stem) = name.strip_suffix(".idx") + && !live.contains(stem) + { + let _ = tokio::fs::remove_file(e.path()).await; } } } @@ -194,10 +194,8 @@ impl RemotePacks { .join("objects") .join("pack") .join(format!("pack-{}.idx", p.checksum)); - if installed.is_file() { - if std::fs::hard_link(&installed, &dest).is_err() { - let _ = std::fs::copy(&installed, &dest); - } + if installed.is_file() && std::fs::hard_link(&installed, &dest).is_err() { + let _ = std::fs::copy(&installed, &dest); } } let done = Arc::new(AtomicU64::new(0)); @@ -228,7 +226,10 @@ impl RemotePacks { let reporter = reporter.clone(); let throttle = throttle.clone(); tasks.push(tokio::spawn(async move { - let _permit = sem.acquire().await.unwrap(); + let _permit = sem + .acquire() + .await + .map_err(|e| WalError::Corrupt(e.to_string()))?; let tmp = dir.join(format!("{}.idx.tmp", p.checksum)); let dest = dir.join(format!("{}.idx", p.checksum)); let cb = |delta: u64, _t: u64| { @@ -273,7 +274,7 @@ impl RemotePacks { let size = if p.pack_size > 0 { p.pack_size } else { - store.head(&key).await?.map(|m| m.size).unwrap_or(0) + store.head(&key).await?.map_or(0, |m| m.size) }; packs.push(RemotePack { checksum: p.checksum.clone(), @@ -296,7 +297,7 @@ impl RemotePacks { objects: moka::sync::Cache::builder() .max_capacity(object_cache_bytes.max(8 * 1024 * 1024)) .weigher(|_k: &(usize, u64), v: &Arc| { - (v.data.len() + 64).clamp(1, u32::MAX as usize) as u32 + u32::try_from(v.data.len().saturating_add(64)).unwrap_or(u32::MAX) }) .build(), hash, @@ -315,7 +316,10 @@ impl RemotePacks { self.packs.iter().map(|p| p.checksum.as_str()).collect() } pub fn total_objects(&self) -> u64 { - self.packs.iter().map(|p| p.idx.num_objects() as u64).sum() + self.packs + .iter() + .map(|p| u64::from(p.idx.num_objects())) + .sum() } /// Locate an object: (pack index, pack offset). @@ -375,7 +379,10 @@ impl RemotePacks { let (entry, _) = self.read_entry_header(cur.0, cur.1).await?; match entry.header { Header::Blob | Header::Tree | Header::Commit | Header::Tag => { - let kind = entry.header.as_kind().expect("base kind"); + let kind = entry + .header + .as_kind() + .ok_or_else(|| WalError::Corrupt("expected base object kind".into()))?; return Ok(Some((kind, size.unwrap_or(entry.decompressed_size)))); } Header::OfsDelta { base_distance } => { @@ -411,12 +418,13 @@ impl RemotePacks { if let Some(o) = self.objects.get(&(pi, off)) { return Ok(o); } - let span = tracing::debug_span!("remote.decode", repo = %self.repo, pack = %self.packs[pi].checksum, offset = off, oid_kind = tracing::field::Empty, chain = tracing::field::Empty); + let span = tracing::debug_span!("remote.decode", repo = %self.repo, pack = %self.packs.get(pi).ok_or_else(|| WalError::Corrupt("pack index out of bounds".into()))?.checksum, offset = off, oid_kind = tracing::field::Empty, chain = tracing::field::Empty); let r = self.decode_inner(pi, off).instrument(span.clone()).await; if let Ok((o, chain)) = &r { span.record("oid_kind", format!("{:?}", o.kind).to_lowercase()); span.record("chain", *chain); - metrics::histogram!("walgit_remote_delta_chain").record(*chain as f64); + metrics::histogram!("walgit_remote_delta_chain") + .record(f64::from(u32::try_from(*chain).unwrap_or(u32::MAX))); } r.map(|(o, _)| o) } @@ -436,7 +444,10 @@ impl RemotePacks { match entry.header { Header::Blob | Header::Tree | Header::Commit | Header::Tag => { let o = Arc::new(Obj { - kind: entry.header.as_kind().expect("base kind"), + kind: entry + .header + .as_kind() + .ok_or_else(|| WalError::Corrupt("expected base object kind".into()))?, data: Bytes::from(data), }); self.objects.insert(cur, o.clone()); @@ -474,8 +485,11 @@ impl RemotePacks { /// Bytes `[off, off+len)` of pack `pi`, assembled from cached blocks /// (missing blocks fetched concurrently). async fn read_at(&self, pi: usize, off: u64, len: u64) -> Result { - let p = &self.packs[pi]; - let end = (off + len).min(p.size); + let p = &self + .packs + .get(pi) + .ok_or_else(|| WalError::Corrupt("pack index out of bounds".into()))?; + let end = off.saturating_add(len).min(p.size); if off >= end { return Ok(Bytes::new()); } @@ -487,17 +501,31 @@ impl RemotePacks { }); let blocks = futures::future::try_join_all(futs).await?; if blocks.len() == 1 { - let b = &blocks[0]; - let s = (off - first * BLOCK_SIZE) as usize; - let e = (end - first * BLOCK_SIZE) as usize; - return Ok(b.slice(s..e)); + let b = blocks + .first() + .ok_or_else(|| WalError::Corrupt("missing range block".into()))?; + let s = usize::try_from(off - first * BLOCK_SIZE) + .map_err(|e| WalError::Corrupt(e.to_string()))?; + let e = usize::try_from(end - first * BLOCK_SIZE) + .map_err(|e| WalError::Corrupt(e.to_string()))?; + return Ok(b.slice_ref( + b.get(s..e) + .ok_or_else(|| WalError::Corrupt("short range block".into()))?, + )); } - let mut out = Vec::with_capacity((end - off) as usize); + let mut out = Vec::with_capacity( + usize::try_from(end - off).map_err(|e| WalError::Corrupt(e.to_string()))?, + ); for (i, b) in blocks.iter().enumerate() { let bstart = (first + i as u64) * BLOCK_SIZE; - let s = off.saturating_sub(bstart) as usize; - let e = (end - bstart).min(b.len() as u64) as usize; - out.extend_from_slice(&b[s..e]); + let s = usize::try_from(off.saturating_sub(bstart)) + .map_err(|e| WalError::Corrupt(e.to_string()))?; + let e = usize::try_from((end - bstart).min(b.len() as u64)) + .map_err(|e| WalError::Corrupt(e.to_string()))?; + out.extend_from_slice( + b.get(s..e) + .ok_or_else(|| WalError::Corrupt("short range block".into()))?, + ); } Ok(Bytes::from(out)) } @@ -531,10 +559,15 @@ impl RemotePacks { head: Bytes, ) -> Result, WalError> { use flate2::{Decompress, FlushDecompress, Status}; - let p = &self.packs[pi]; - let size = entry.decompressed_size as usize; + let p = &self + .packs + .get(pi) + .ok_or_else(|| WalError::Corrupt("pack index out of bounds".into()))?; + let size = usize::try_from(entry.decompressed_size) + .map_err(|e| WalError::Corrupt(e.to_string()))?; let data_off = entry.data_offset; - let header_len = (data_off - entry.pack_offset()) as usize; + let header_len = usize::try_from(data_off - entry.pack_offset()) + .map_err(|e| WalError::Corrupt(e.to_string()))?; // Prefetch: blocks from data_off through data_off + size (+ slack), bounded. { let guess_end = @@ -581,7 +614,8 @@ impl RemotePacks { entry.pack_offset() )) })?; - let consumed = (z.total_in() - before_in) as usize; + let consumed = usize::try_from(z.total_in() - before_in) + .map_err(|e| WalError::Corrupt(e.to_string()))?; pos += consumed as u64; chunk = chunk.slice(consumed..); if out.len() >= size || status == Status::StreamEnd { @@ -614,7 +648,7 @@ fn varint(d: &[u8], mut i: usize) -> Result<(u64, usize), &'static str> { loop { let b = *d.get(i).ok_or("delta header truncated")?; i += 1; - v |= ((b & 0x7f) as u64) << shift; + v |= u64::from(b & 0x7f) << shift; shift += 7; if b & 0x80 == 0 { return Ok((v, i)); @@ -634,13 +668,13 @@ fn delta_result_size(delta: &[u8]) -> Result { /// Apply a git delta (`base` + `delta` instructions → result). pub fn apply_delta(base: &[u8], delta: &[u8]) -> Result, &'static str> { let (base_size, i) = varint(delta, 0)?; - if base_size as usize != base.len() { + if base_size != base.len() as u64 { return Err("delta base size mismatch"); } let (res_size, mut i) = varint(delta, i)?; - let mut out = Vec::with_capacity(res_size as usize); - while i < delta.len() { - let cmd = delta[i]; + let mut out = + Vec::with_capacity(usize::try_from(res_size).map_err(|_| "delta result too large")?); + while let Some(&cmd) = delta.get(i) { i += 1; if cmd & 0x80 != 0 { let mut ofs: u64 = 0; @@ -648,7 +682,7 @@ pub fn apply_delta(base: &[u8], delta: &[u8]) -> Result, &'static str> { let mut nb = |shift: u32| -> Result { let b = *delta.get(i).ok_or("delta copy truncated")?; i += 1; - Ok((b as u64) << shift) + Ok(u64::from(b) << shift) }; if cmd & 0x01 != 0 { ofs |= nb(0)?; @@ -675,10 +709,15 @@ pub fn apply_delta(base: &[u8], delta: &[u8]) -> Result, &'static str> { size = 0x10000; } let end = ofs.checked_add(size).ok_or("delta copy overflow")?; - if end as usize > base.len() { + if end > base.len() as u64 { return Err("delta copy out of base bounds"); } - out.extend_from_slice(&base[ofs as usize..end as usize]); + let start = usize::try_from(ofs).map_err(|_| "delta copy offset too large")?; + let end = usize::try_from(end).map_err(|_| "delta copy end too large")?; + out.extend_from_slice( + base.get(start..end) + .ok_or("delta copy out of base bounds")?, + ); } else if cmd != 0 { let n = cmd as usize; let src = delta.get(i..i + n).ok_or("delta insert truncated")?; @@ -694,35 +733,25 @@ pub fn apply_delta(base: &[u8], delta: &[u8]) -> Result, &'static str> { Ok(out) } +#[expect( + clippy::cast_precision_loss, + reason = "Human-readable byte counts intentionally round to one decimal place" +)] pub fn human_bytes(n: u64) -> String { const U: [&str; 5] = ["B", "KB", "MB", "GB", "TB"]; let mut v = n as f64; - let mut i = 0; - while v >= 1024.0 && i < U.len() - 1 { + let mut unit = "B"; + for next in U.iter().skip(1) { + if v < 1024.0 { + break; + } v /= 1024.0; - i += 1; + unit = next; } - if i == 0 { + if unit == "B" { format!("{n} B") } else { - format!("{v:.1} {}", U[i]) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn delta_roundtrip_insert_and_copy() { - let base = b"hello world, this is the base object"; - // header: base size, result size; then copy 0..5 from base, insert "!!", copy 5..12 - let mut d = vec![base.len() as u8, 5 + 2 + 7]; - d.extend([0x90, 5]); // copy ofs=0 (no ofs bytes), size=5 (0x10 flag) - d.extend([2, b'!', b'!']); - d.extend([0x91, 5, 7]); // copy ofs=5 size=7 - let out = apply_delta(base, &d).unwrap(); - assert_eq!(out, b"hello!! world,"); + format!("{v:.1} {unit}") } } @@ -772,8 +801,10 @@ impl walgit_git::ObjectFaulter for Faulter { ); Box::pin( async move { - self.rounds.fetch_add(1, Ordering::Relaxed); const PAR: usize = 32; + + self.rounds.fetch_add(1, Ordering::Relaxed); + let mut n = 0usize; for chunk in oids.chunks(PAR) { let results = @@ -803,3 +834,23 @@ impl walgit_git::ObjectFaulter for Faulter { ) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn delta_roundtrip_insert_and_copy() { + let base = b"hello world, this is the base object"; + // header: base size, result size; then copy 0..5 from base, insert "!!", copy 5..12 + let mut d = vec![ + u8::try_from(base.len()).expect("test base is shorter than 256 bytes"), + 5 + 2 + 7, + ]; + d.extend([0x90, 5]); // copy ofs=0 (no ofs bytes), size=5 (0x10 flag) + d.extend([2, b'!', b'!']); + d.extend([0x91, 5, 7]); // copy ofs=5 size=7 + let out = apply_delta(base, &d).unwrap(); + assert_eq!(out, b"hello!! world,"); + } +} diff --git a/crates/walgit-wal/src/state.rs b/crates/walgit-wal/src/state.rs index 779f0cf..a150b94 100644 --- a/crates/walgit-wal/src/state.rs +++ b/crates/walgit-wal/src/state.rs @@ -1,4 +1,4 @@ -//! Persistent local state for a RepoHandle, stored in the repo dir so restarts +//! Persistent local state for a `RepoHandle`, stored in the repo dir so restarts //! skip already-applied log entries. use std::path::Path; @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use crate::error::WalError; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct RepoState { /// Opaque version string of the last manifest we applied. pub manifest_version: Option, @@ -39,19 +39,6 @@ impl RepoState { } } -impl Default for RepoState { - fn default() -> Self { - RepoState { - manifest_version: None, - applied_seq: 0, - revision: 0, - packs_revision: 0, - pending_pack_removals: Vec::new(), - remote_served: Vec::new(), - } - } -} - impl RepoState {} const STATE_FILE: &str = "walgit-state.json"; diff --git a/crates/walgit-wal/src/sync.rs b/crates/walgit-wal/src/sync.rs index 978a10d..c9134e9 100644 --- a/crates/walgit-wal/src/sync.rs +++ b/crates/walgit-wal/src/sync.rs @@ -1,4 +1,4 @@ -//! sync() implementation: freshness check, catch-up, materialization. +//! `sync()` implementation: freshness check, catch-up, materialization. use std::sync::Arc; @@ -11,13 +11,13 @@ use walgit_proto::v1::{EntryKind, LogEntry, Manifest, PackRef, RefSnapshot}; use walgit_store::{GetOptions, GetResult, ObjectStore, Prefixed, Version}; /// A read guard held for the lifetime of a request. While any guard is alive -/// no pack is removed locally (the inner RwLock read guard prevents it). +/// no pack is removed locally (the inner `RwLock` read guard prevents it). pub struct ReadGuard<'a> { pub(crate) _guard: tokio::sync::RwLockReadGuard<'a, ()>, pub(crate) handle: &'a super::handle::RepoHandle, } -impl<'a> ReadGuard<'a> { +impl ReadGuard<'_> { pub fn manifest(&self) -> Arc { self.handle.manifest.read().clone() } @@ -82,28 +82,28 @@ pub(crate) enum SyncOutcome { Unchanged, Changed { meta_version: Version, - manifest: Manifest, + manifest: std::sync::Arc, }, } /// Perform a conditional GET on manifest.pb and return the outcome. pub(crate) async fn freshness_check( store: &Prefixed, - known: &Option, + known: Option<&Version>, ) -> Result { match known { Some(v) => match get_message_if_changed::(store, keys::MANIFEST, v).await? { None => Ok(SyncOutcome::Unchanged), Some((meta, manifest)) => Ok(SyncOutcome::Changed { meta_version: meta.version, - manifest, + manifest: std::sync::Arc::new(manifest), }), }, None => match get_message::(store, keys::MANIFEST).await? { None => Err(WalError::NotFound), Some((meta, manifest)) => Ok(SyncOutcome::Changed { meta_version: meta.version, - manifest, + manifest: std::sync::Arc::new(manifest), }), }, } @@ -320,7 +320,7 @@ fn side_files(pack: &PackRef) -> [(bool, &'static str, String); 3] { /// NIC's worth), with bounded memory (PAR * CHUNK). /// `progress(delta_bytes, total_bytes)` is called as chunks land (callers /// throttle). `known_size` skips the happy-path HEAD (ROUNDTRIPS: HEAD ≈ GET; -/// PackRef already carries pack/idx sizes). +/// `PackRef` already carries pack/idx sizes). pub(crate) type ProgressFn<'a> = &'a (dyn Fn(u64, u64) + Send + Sync); fn nonzero(n: u64) -> Option { @@ -380,7 +380,9 @@ pub(crate) async fn download_object( let file = std::fs::File::create(dest)?; file.set_len(size)?; let file = std::sync::Arc::new(file); - let starts: Vec = (0..size).step_by(CHUNK as usize).collect(); + let starts: Vec = (0..size) + .step_by(usize::try_from(CHUNK).map_err(|e| WalError::Corrupt(e.to_string()))?) + .collect(); let report = &report; futures::stream::iter(starts) .map(|start| { @@ -402,7 +404,11 @@ pub(crate) async fn download_object( return Err(WalError::Corrupt(format!("unexpected 304 for {key}"))); } }; - let bytes = walgit_store::util::collect(body, (end - start) as usize).await?; + let bytes = walgit_store::util::collect( + body, + usize::try_from(end - start).map_err(|e| WalError::Corrupt(e.to_string()))?, + ) + .await?; if bytes.len() as u64 != end - start { return Err(WalError::Corrupt(format!( "short range read for {key}: {}..{} got {}", @@ -438,7 +444,7 @@ pub(crate) async fn apply_delta( // If we have a checkpoint and haven't loaded it yet, load its refs. Its // packs are a subset of `Manifest.packs` and are reconciled below. - let checkpoint_seq = new_manifest.checkpoint.as_ref().map(|c| c.seq).unwrap_or(0); + let checkpoint_seq = new_manifest.checkpoint.as_ref().map_or(0, |c| c.seq); let need_checkpoint_load = checkpoint_seq > 0 && current_state.applied_seq < checkpoint_seq; // The checkpoint's times feed `first_state_time` / `refs_as_of`; old refs @@ -563,10 +569,12 @@ pub(crate) async fn reconcile_packs_inner( } { let mut st = handle.state.lock(); - st.remote_served = remote_served.clone(); + st.remote_served.clone_from(&remote_served); } - let remote_set: std::collections::HashSet<&str> = - remote_served.iter().map(|s| s.as_str()).collect(); + let remote_set: std::collections::HashSet<&str> = remote_served + .iter() + .map(std::string::String::as_str) + .collect(); // History packs (D18) are an accelerator, not a requirement: a fetch can // be served from the linked/remote base right away. They are installed by @@ -613,7 +621,7 @@ pub(crate) async fn reconcile_packs_inner( tracing::info!(repo = %handle.id, pack = %p.checksum, ext, "side-file installed for an installed pack"); } Err(e) => { - tracing::warn!(repo = %handle.id, pack = %p.checksum, ext, error = %e, "side-file download failed") + tracing::warn!(repo = %handle.id, pack = %p.checksum, ext, error = %e, "side-file download failed"); } } } @@ -684,7 +692,10 @@ pub(crate) async fn reconcile_packs_inner( let link_to = link_target(&p); tasks.push(tokio::spawn( async move { - let _permit = sem.acquire().await.unwrap(); + let _permit = sem + .acquire() + .await + .map_err(|e| WalError::Corrupt(e.to_string()))?; // Per-object progress arrives as absolute (done,total); turn it // into deltas for the shared counter. let cb = |delta: u64, _t: u64| { @@ -739,19 +750,16 @@ pub(crate) async fn reconcile_packs_inner( }) .collect::>()?; if !to_remove.is_empty() { - match handle.rw.try_write() { - Ok(_w) => { - for (_, oid) in &to_remove { - if local.pack_path(oid).exists() { - local.remove_pack(oid)?; - removed += 1; - } + if let Ok(_w) = handle.rw.try_write() { + for (_, oid) in &to_remove { + if local.pack_path(oid).exists() { + local.remove_pack(oid)?; + removed += 1; } } - Err(_) => { - tracing::info!(repo = %handle.id, packs = to_remove.len(), "superseded packs kept for now: readers active; retried on the next sync"); - still_pending.extend(to_remove.iter().map(|(s, _)| s.clone())); - } + } else { + tracing::info!(repo = %handle.id, packs = to_remove.len(), "superseded packs kept for now: readers active; retried on the next sync"); + still_pending.extend(to_remove.iter().map(|(s, _)| s.clone())); } } span.record("removed", removed); @@ -792,10 +800,10 @@ pub(crate) async fn maintain_commit_graph( Ok(Ok(true)) => base_changed = true, Ok(Ok(false)) => {} Ok(Err(e)) => { - tracing::warn!(pack = %p.checksum, error = %e, "commit-graph base install failed") + tracing::warn!(pack = %p.checksum, error = %e, "commit-graph base install failed"); } Err(e) => { - tracing::warn!(pack = %p.checksum, error = %e, "commit-graph base install task failed") + tracing::warn!(pack = %p.checksum, error = %e, "commit-graph base install task failed"); } } } @@ -831,11 +839,11 @@ pub(crate) async fn maintain_commit_graph( { tracing::warn!(repo = %handle.id, error = %e, "commit-graph update failed"); } else { - tracing::info!(repo = %handle.id, packs = packs.len(), ms = started.elapsed().as_millis() as u64, "commit-graph updated"); + tracing::info!(repo = %handle.id, packs = packs.len(), ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), "commit-graph updated"); } } -/// Replay log entries in (from_seq, to_seq] from the manifest's log segments. +/// Replay log entries in (`from_seq`, `to_seq`] from the manifest's log segments. pub(crate) async fn replay_log( handle: &super::handle::RepoHandle, manifest: &Manifest, @@ -867,9 +875,14 @@ pub(crate) async fn replay_log( async move { let res = store.get(&key, GetOptions::default()).await?; Ok::, WalError>(match res { - GetResult::Object { meta, body } => { - Some(walgit_store::util::collect(body, meta.size as usize).await?) - } + GetResult::Object { meta, body } => Some( + walgit_store::util::collect( + body, + usize::try_from(meta.size) + .map_err(|e| WalError::Corrupt(e.to_string()))?, + ) + .await?, + ), GetResult::NotModified { .. } => None, }) } @@ -946,9 +959,8 @@ pub(crate) fn apply_entries( EntryKind::Compact => { supersedes.extend(entry.supersedes.iter().cloned()); } - EntryKind::Checkpoint => {} + EntryKind::Checkpoint | EntryKind::Settings => {} // Settings live on the manifest; the entry is history only. - EntryKind::Settings => {} EntryKind::Unspecified => { tracing::warn!(seq = entry.seq, "unspecified log entry kind, skipping"); } @@ -992,6 +1004,46 @@ pub(crate) async fn materialize_from_scratch( .await } +/// The **bulk runtime**: a small dedicated tokio runtime (own worker threads) +/// that runs pack materialization (striped downloads, 32 MiB chunk copies, +/// tmpfs writes, install renames, gix reopen, commit-graph/midx subprocess +/// waits). Whatever inside that path is CPU-heavy or secretly blocking can +/// only delay other bulk work — request workers on the main runtime keep +/// serving refs in milliseconds (prod 2026-08-20: the main runtime stalled +/// 2.6–43 s repeatedly for the whole duration of one repo's 7.5 GB + another's +/// 12 GB materializations; the watchdog caught it, the cause hid among a dozen +/// candidates; isolation makes the question moot). +static BULK_RUNTIME: std::sync::OnceLock> = + std::sync::OnceLock::new(); + +fn bulk_runtime() -> Result<&'static tokio::runtime::Runtime, WalError> { + BULK_RUNTIME + .get_or_init(|| { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .thread_name("walgit-bulk") + .enable_all() + .build() + }) + .as_ref() + .map_err(|e| std::io::Error::new(e.kind(), format!("bulk runtime: {e}")).into()) +} + +/// Run `fut` on the bulk runtime and await its result from the caller's +/// runtime. The future must be `'static + Send` (use `Arc`). +pub(crate) async fn on_bulk_runtime( + fut: impl std::future::Future> + Send + 'static, +) -> Result { + let span = tracing::Span::current(); + let (tx, rx) = tokio::sync::oneshot::channel(); + bulk_runtime()?.spawn(async move { + let r = fut.instrument(span).await; + let _ = tx.send(r); + }); + rx.await + .map_err(|_| WalError::Corrupt("bulk runtime task dropped".into()))? +} + #[cfg(test)] mod download_tests { use super::download_object; @@ -1002,12 +1054,12 @@ mod download_tests { // > CHUNK (32 MiB) so the ranged/striped path runs, with a ragged tail. let size = 70 * 1024 * 1024 + 12345; let mut data = vec![0u8; size]; - let mut x: u64 = 0x9E3779B97F4A7C15; - for b in data.iter_mut() { + let mut x: u64 = 0x9E37_79B9_7F4A_7C15; + for b in &mut data { x ^= x << 13; x ^= x >> 7; x ^= x << 17; - *b = x as u8; + *b = x.to_le_bytes()[0]; } let store = MemoryStore::shared(); store @@ -1032,40 +1084,3 @@ mod download_tests { assert_eq!(std::fs::read(&small).unwrap(), b"tiny"); } } - -/// The **bulk runtime**: a small dedicated tokio runtime (own worker threads) -/// that runs pack materialization (striped downloads, 32 MiB chunk copies, -/// tmpfs writes, install renames, gix reopen, commit-graph/midx subprocess -/// waits). Whatever inside that path is CPU-heavy or secretly blocking can -/// only delay other bulk work — request workers on the main runtime keep -/// serving refs in milliseconds (prod 2026-08-20: the main runtime stalled -/// 2.6–43 s repeatedly for the whole duration of one repo's 7.5 GB + another's -/// 12 GB materializations; the watchdog caught it, the cause hid among a dozen -/// candidates; isolation makes the question moot). -static BULK_RUNTIME: std::sync::OnceLock = std::sync::OnceLock::new(); - -fn bulk_runtime() -> &'static tokio::runtime::Runtime { - BULK_RUNTIME.get_or_init(|| { - tokio::runtime::Builder::new_multi_thread() - .worker_threads(4) - .thread_name("walgit-bulk") - .enable_all() - .build() - .expect("bulk runtime") - }) -} - -/// Run `fut` on the bulk runtime and await its result from the caller's -/// runtime. The future must be `'static + Send` (use `Arc`). -pub(crate) async fn on_bulk_runtime( - fut: impl std::future::Future> + Send + 'static, -) -> Result { - let span = tracing::Span::current(); - let (tx, rx) = tokio::sync::oneshot::channel(); - bulk_runtime().spawn(async move { - let r = fut.instrument(span).await; - let _ = tx.send(r); - }); - rx.await - .map_err(|_| WalError::Corrupt("bulk runtime task dropped".into()))? -} diff --git a/crates/walgit-wal/src/tasks.rs b/crates/walgit-wal/src/tasks.rs index 96caaae..b4c3535 100644 --- a/crates/walgit-wal/src/tasks.rs +++ b/crates/walgit-wal/src/tasks.rs @@ -30,6 +30,13 @@ const KEEP_RECORDS: usize = 30; const KEEP_LOG: usize = 60; const REPLAY: usize = 200; +/// Replayed packets, future packets, and the terminal outcome if already finished. +pub type TaskAttachment = ( + Vec, + tokio::sync::broadcast::Receiver, + Option>, +); + #[derive(Serialize, Clone, Debug)] pub struct TaskRecord { pub id: String, @@ -106,13 +113,7 @@ impl TaskState { self.record.lock().clone() } /// Subscribe + snapshot of everything so far (no gap, no duplicates). - pub fn attach( - &self, - ) -> ( - Vec, - tokio::sync::broadcast::Receiver, - Option>, - ) { + pub fn attach(&self) -> TaskAttachment { let replay = self.replay.lock(); let rx = self.tx.subscribe(); let outcome = self.outcome.lock().clone(); @@ -157,7 +158,8 @@ impl TaskState { Progress::Progress { .. } => rec.progress = Some(p.clone()), Progress::Task { .. } => {} } - rec.elapsed_ms = self.started_at.elapsed().as_millis() as u64; + rec.elapsed_ms = + u64::try_from(self.started_at.elapsed().as_millis()).unwrap_or(u64::MAX); } { let mut replay = self.replay.lock(); @@ -389,15 +391,16 @@ impl Tasks { let record = { let mut rec = state.record.lock(); rec.finished = Some(now_rfc3339()); - rec.elapsed_ms = state.started_at.elapsed().as_millis() as u64; + rec.elapsed_ms = + u64::try_from(state.started_at.elapsed().as_millis()).unwrap_or(u64::MAX); match &outcome { Ok((summary, _)) => { rec.ok = Some(true); - rec.summary = summary.clone(); + rec.summary.clone_from(summary); } Err((_, msg)) => { rec.ok = Some(false); - rec.summary = msg.clone(); + rec.summary.clone_from(msg); } } rec.clone() @@ -431,7 +434,7 @@ impl Tasks { pick_from .as_ref() .and_then(|v| v.get(k)) - .and_then(|v| v.as_u64()) + .and_then(serde_json::Value::as_u64) }; let (bytes, objects) = (pick("bytes").or_else(|| pick("size")), pick("objects")); let ok = record.ok.unwrap_or(false); @@ -444,7 +447,7 @@ impl Tasks { }; tracing::info!(repo = %record.repo, kind = %record.kind, id = %record.id, ok, outcome, elapsed_ms = record.elapsed_ms, bytes, objects, "task finished: {}", record.summary); metrics::counter!("walgit_tasks_finished_total", "kind" => record.kind.clone(), "ok" => ok.to_string()).increment(1); - metrics::histogram!("walgit_task_duration_seconds", "kind" => record.kind.clone(), "ok" => ok.to_string()).record(record.elapsed_ms as f64 / 1000.0); + metrics::histogram!("walgit_task_duration_seconds", "kind" => record.kind.clone(), "ok" => ok.to_string()).record(std::time::Duration::from_millis(record.elapsed_ms).as_secs_f64()); record } diff --git a/crates/walgit-wal/tests/wal.rs b/crates/walgit-wal/tests/wal.rs index ef7b34e..3779d85 100644 --- a/crates/walgit-wal/tests/wal.rs +++ b/crates/walgit-wal/tests/wal.rs @@ -1,6 +1,15 @@ +#![allow( + clippy::cast_sign_loss, + clippy::field_reassign_with_default, + clippy::unreadable_literal, + clippy::zombie_processes +)] +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::unwrap_used)] + //! Integration tests for walgit-wal. //! -//! Uses MemoryStore + real LocalRepo tempdir + upstream git to create +//! Uses `MemoryStore` + real `LocalRepo` tempdir + upstream git to create //! objects/packs. use std::collections::HashMap; @@ -832,7 +841,7 @@ async fn test_orphan_log_invisible_and_cleaned() { .store() .put( &orphan_key, - bytes::Bytes::from(orphan_bytes).into(), + orphan_bytes.into(), walgit_store::PutMode::Create.into(), ) .await @@ -950,7 +959,9 @@ async fn test_serve_level_links_base_from_store_mount() { x ^= x << 13; x ^= x >> 7; x ^= x << 17; - body.push_str(&format!("{x:016x}")); + { + let _ = std::fmt::Write::write_fmt(&mut body, format_args!("{x:016x}")); + }; } let c = work.commit(&format!("base_{i}"), &body); let pack = if prev.is_empty() { @@ -1191,7 +1202,7 @@ fn checkpoint_due_triggers() { use walgit_wal::{CheckpointTrigger, checkpoint_due}; let mut cfg = walgit_config::WalConfig::default(); cfg.snapshot_every_entries = 10; - cfg.checkpoint_interval = Duration::from_secs(3600); + cfg.checkpoint_interval = Duration::from_hours(1); cfg.checkpoint_tail_bytes = walgit_config::ByteSize::kib(1); let seg = |first: u64, last: u64, size: u64| LogSegmentRef { key: String::new(), @@ -1221,7 +1232,7 @@ fn checkpoint_due_triggers() { assert_eq!(checkpoint_due(&m, &cfg), Some(CheckpointTrigger::TailBytes)); m.log_segments = vec![seg(1, 3, 100)]; - let old = std::time::SystemTime::now() - Duration::from_secs(7200); + let old = std::time::SystemTime::now() - Duration::from_hours(2); m.updated_at = Some(walgit_proto::time::from_system(old)); assert_eq!( checkpoint_due(&m, &cfg), @@ -1271,6 +1282,8 @@ fn checkpoint_due_triggers() { /// from checkpoint + tail. #[tokio::test] async fn test_checkpoint_from_refs_level_instance() { + use prost::Message; + let cache = tempfile::tempdir().unwrap(); let store = MemoryStore::shared(); let registry = Registry::new(store.clone(), Arc::new(make_config(cache.path(), 0))); @@ -1349,7 +1362,7 @@ async fn test_checkpoint_from_refs_level_instance() { assert_eq!(handle2.checkpoint_due(), None); // The checkpoint object carries the pack inventory with side-file flags. - use prost::Message; + let (_, bytes) = walgit_store::ObjectStoreExt::get_bytes(handle2.store(), &cp.key) .await .unwrap() @@ -1393,7 +1406,9 @@ async fn test_serve_level_remote_serves_base_without_mount() { x ^= x << 13; x ^= x >> 7; x ^= x << 17; - body.push_str(&format!("{x:016x}")); + { + let _ = std::fmt::Write::write_fmt(&mut body, format_args!("{x:016x}")); + }; } let c = work.commit(&format!("base_{i}"), &body); let pack = if prev.is_empty() { @@ -1524,7 +1539,7 @@ async fn test_serve_level_remote_serves_base_without_mount() { assert_eq!(stats.objects, 3, "{stats:?}"); let (faulted, rounds) = faulter.stats(); assert!( - faulted >= 1 && faulted <= 3, + (1..=3).contains(&faulted), "faulted {faulted} (parent commit + root tree)" ); assert!(rounds <= 3); @@ -1600,6 +1615,8 @@ async fn test_annotate_pack_retrofits_commit_graph() { /// so `sync_refs()` on a cold instance answers while packs still download. #[tokio::test] async fn test_refs_sync_is_not_blocked_by_pack_materialization() { + use futures::StreamExt; + let cache = tempfile::tempdir().unwrap(); let store = MemoryStore::shared(); let registry = Registry::new(store.clone(), Arc::new(make_config(cache.path(), 0))); @@ -1634,7 +1651,7 @@ async fn test_refs_sync_is_not_blocked_by_pack_materialization() { inner.latency = Some(Duration::from_millis(150)); } // Copy the data over. - use futures::StreamExt; + let mut keys = store.list("", None); while let Some(m) = keys.next().await { let m = m.unwrap(); @@ -1707,7 +1724,9 @@ async fn test_history_pack_keeps_tree_walks_local() { x ^= x << 13; x ^= x >> 7; x ^= x << 17; - body.push_str(&format!("{x:016x}")); + { + let _ = std::fmt::Write::write_fmt(&mut body, format_args!("{x:016x}")); + }; } std::fs::create_dir_all(work.path().join(format!("d{i}/sub"))).unwrap(); std::fs::write(work.path().join(format!("d{i}/sub/big.bin")), &body).unwrap(); @@ -1930,7 +1949,7 @@ async fn test_history_pack_keeps_tree_walks_local() { /// A long-lived read guard (a clone streaming for minutes) plus a pack /// removal that wants the write lock must not block new refs-level syncs: -/// a queued writer on a tokio RwLock stalls every new reader (prod: info/refs +/// a queued writer on a tokio `RwLock` stalls every new reader (prod: info/refs /// waited 60–680 s behind one 24-minute clone). Removal is try-only now. #[tokio::test] async fn test_refs_sync_never_waits_behind_a_long_read_guard() { @@ -2341,6 +2360,10 @@ async fn test_checkpoint_carries_first_state_and_as_of() { /// entry (every slot in between planned as "unavailable" in prod). #[tokio::test] async fn test_first_state_time_uses_the_checkpoint_when_early_entries_are_untimestamped() { + use walgit_store::ObjectStoreExt; + + use prost::Message; + let cache = tempfile::tempdir().unwrap(); let store = MemoryStore::shared(); let registry = Registry::new(store.clone(), Arc::new(make_config(cache.path(), 0))); @@ -2378,8 +2401,7 @@ async fn test_first_state_time_uses_the_checkpoint_when_early_entries_are_untime // Rewrite the bucket the way 2026-08-19 wrote it: checkpoint ref without // first_state_at/as_of, created on 08-02; log entry 2 without created_at. - use prost::Message; - use walgit_store::ObjectStoreExt; + let mkey = format!("{}{}", id.store_prefix(), walgit_proto::keys::MANIFEST); let (_, bytes) = store.get_bytes(&mkey).await.unwrap().unwrap(); let mut m = walgit_proto::v1::Manifest::decode(bytes.as_ref()).unwrap(); @@ -2446,6 +2468,10 @@ async fn test_first_state_time_uses_the_checkpoint_when_early_entries_are_untime /// state" and the bundler cut it from today's main (prod 2026-08-21 04:2xZ). #[tokio::test] async fn test_checkpoint_times_come_from_the_object_when_the_ref_has_none() { + use walgit_store::ObjectStoreExt; + + use prost::Message; + let cache = tempfile::tempdir().unwrap(); let store = MemoryStore::shared(); let registry = Registry::new(store.clone(), Arc::new(make_config(cache.path(), 0))); @@ -2482,8 +2508,7 @@ async fn test_checkpoint_times_come_from_the_object_when_the_ref_has_none() { .unwrap(); // Strip the ref's times (08-19 import shape); stamp the object 08-19 21:33Z. - use prost::Message; - use walgit_store::ObjectStoreExt; + let mkey = format!("{}{}", id.store_prefix(), walgit_proto::keys::MANIFEST); let (_, bytes) = store.get_bytes(&mkey).await.unwrap().unwrap(); let mut m = walgit_proto::v1::Manifest::decode(bytes.as_ref()).unwrap(); From 202ebcc027840b8d9c5d4a00b9b4a826eb1973bb Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Thu, 27 Aug 2026 00:24:40 -0400 Subject: [PATCH 10/28] Require an explicit issuer in oidc mode AuthConfig::default set issuer to https://accounts.google.com, so a config with mode = "oidc", an allowlist, an OAuth client and a session secret but no issuer passed config check and the server then ran discovery against Google, fetched Google's JWKS and validated every ID token as a Google identity (crates/walgit-server/src/auth.rs:150 and :777). The default is now an empty string, which the check at crates/walgit-config/src/lib.rs:1511-1515 rejects, so oidc mode fails closed until the operator names the issuer as AGENTS.md section 1.3 requires. That check sits inside the mode == Oidc arm opened at lib.rs:1502, so none and token mode are untouched. tests::auth_modes_validate_fail_closed proves it: a minimal oidc config without an issuer now fails on the issuer message, the two cases that used to ride on the Google default carry an explicit issuer, and two new assertions show none and token mode validate with the issuer empty. The walgit.example.toml comment near line 53 says the setting is required, and the ID token fixture in auth.rs sets the issuer it signs with. Co-Authored-By: Claude Fable 5 --- crates/walgit-config/src/lib.rs | 18 +++++++++++++----- crates/walgit-server/src/auth.rs | 1 + walgit.example.toml | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/crates/walgit-config/src/lib.rs b/crates/walgit-config/src/lib.rs index e5c71df..24ad54c 100644 --- a/crates/walgit-config/src/lib.rs +++ b/crates/walgit-config/src/lib.rs @@ -146,7 +146,8 @@ pub struct AuthConfig { pub tokens: Vec, /// OIDC issuer (`oidc` mode). Discovery at `/.well-known/openid-configuration` /// supplies the JWKS, authorization and token endpoints. Any compliant provider works - /// (Google, Microsoft Entra, Okta, Auth0, Keycloak, Dex, GitLab, ...). + /// (Google, Microsoft Entra, Okta, Auth0, Keycloak, Dex, GitLab, ...). No default: + /// `Config::validate` refuses `oidc` mode until it is set. pub issuer: String, /// Email domains accepted by `oidc` (the `email` claim, `email_verified` required). pub allowed_domains: Vec, @@ -1050,7 +1051,7 @@ impl Default for AuthConfig { mode: AuthMode::None, anonymous_read: true, tokens: vec![], - issuer: "https://accounts.google.com".into(), + issuer: String::new(), allowed_domains: vec![], allowed_emails: vec![], audiences: vec![], @@ -1954,10 +1955,12 @@ audiences = ["walgit-cli", "https://git.example.com"] err.to_string().contains("needs `token` or `token_env`"), "{err}" ); - // oidc: anonymous_read off, an allowlist, and a way in. - let err = Config::parse("[store]\nbucket = \"b\"\n[server.auth]\nmode = \"oidc\"\nanonymous_read = false\nallowed_domains = [\"example.com\"]\n").unwrap_err(); + // oidc: an issuer, anonymous_read off, an allowlist, and a way in. + let err = Config::parse("[store]\nbucket = \"b\"\n[server.auth]\nmode = \"oidc\"\nanonymous_read = false\nallowed_domains = [\"example.com\"]\noauth_client_id = \"x\"\noauth_client_secret = \"y\"\nsession_secret = \"0123456789abcdef0123456789abcdef\"\n").unwrap_err(); + assert!(err.to_string().contains("issuer"), "{err}"); + let err = Config::parse("[store]\nbucket = \"b\"\n[server.auth]\nmode = \"oidc\"\nissuer = \"https://login.example.com\"\nanonymous_read = false\nallowed_domains = [\"example.com\"]\n").unwrap_err(); assert!(err.to_string().contains("way in"), "{err}"); - let err = Config::parse("[store]\nbucket = \"b\"\n[server.auth]\nmode = \"oidc\"\nanonymous_read = false\nallowed_domains = [\"example.com\"]\noauth_client_id = \"x\"\noauth_client_secret = \"y\"\n").unwrap_err(); + let err = Config::parse("[store]\nbucket = \"b\"\n[server.auth]\nmode = \"oidc\"\nissuer = \"https://login.example.com\"\nanonymous_read = false\nallowed_domains = [\"example.com\"]\noauth_client_id = \"x\"\noauth_client_secret = \"y\"\n").unwrap_err(); assert!(err.to_string().contains("session_secret"), "{err}"); let ok = Config::parse("[store]\nbucket = \"b\"\n[server.auth]\nmode = \"oidc\"\nissuer = \"https://login.example.com\"\nanonymous_read = false\nallowed_domains = [\"example.com\"]\noauth_client_id = \"x\"\noauth_client_secret = \"y\"\nsession_secret = \"0123456789abcdef0123456789abcdef\"\n").unwrap(); assert_eq!(ok.server.auth.issuer, "https://login.example.com"); @@ -1967,6 +1970,11 @@ audiences = ["walgit-cli", "https://git.example.com"] ) .unwrap_err(); assert!(err.to_string().contains("loopback-only"), "{err}"); + // The issuer is an oidc-only requirement: none and token mode validate without one. + let none = Config::parse("[store]\nbucket = \"b\"\n").unwrap(); + assert_eq!(none.server.auth.issuer, ""); + let tok = Config::parse("[store]\nbucket = \"b\"\n[server.auth]\nmode = \"token\"\ntokens = [{ principal = \"ci\", token = \"s\" }]\n").unwrap(); + assert_eq!(tok.server.auth.issuer, ""); } #[test] diff --git a/crates/walgit-server/src/auth.rs b/crates/walgit-server/src/auth.rs index ff43b97..fe8619a 100644 --- a/crates/walgit-server/src/auth.rs +++ b/crates/walgit-server/src/auth.rs @@ -1103,6 +1103,7 @@ GcZ0izY/30012ajdHY+/QK5lsMoxTnn0skdS+spLxaS5ZEO4qvPVb8RAoCkWMMal fn config() -> walgit_config::Config { let mut cfg = walgit_config::Config::default(); cfg.server.auth.mode = AuthMode::Oidc; + cfg.server.auth.issuer = ISSUER.into(); cfg.server.auth.allowed_domains = vec!["Example.com".into()]; cfg.server.auth.audiences = vec![AUD.into()]; cfg.server.auth.anonymous_read = false; diff --git a/walgit.example.toml b/walgit.example.toml index ea161d4..304778b 100644 --- a/walgit.example.toml +++ b/walgit.example.toml @@ -50,7 +50,7 @@ anonymous_read = true # must be false in oidc mode # ] # admin_emails = [] # oidc: emails that may delete repos or PUT/DELETE settings and policy.json # admin_domains = [] # oidc: email domains that may delete repos or PUT/DELETE settings and policy.json -# issuer = "https://id.example.com" # oidc: discovery at /.well-known/openid-configuration +# issuer = "https://id.example.com" # oidc: required, no default. Discovery at /.well-known/openid-configuration # allowed_domains = ["example.com"] # oidc: email domains admitted (email_verified required) # allowed_emails = [] # oidc: individual identities admitted # write_domains = ["example.com"] # oidc: omit to let every admitted identity write From b81b15ae986911344d61e3080678a358d06e6995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E5=BA=86=E4=B8=B0?= <2295878+gqf2008@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:58:13 +0800 Subject: [PATCH 11/28] =?UTF-8?q?fix(ci):=20strip=20ANSI=20escapes=20in=20?= =?UTF-8?q?`just=20warnings`=20before=20matching=20=E2=80=94=20gate=20work?= =?UTF-8?q?s=20again?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI exports CARGO_TERM_COLOR=always, so rustc diagnostics carry ANSI escape prefixes and the warnings recipe's anchored '^warning:' grep never matches: a warning-bearing tree passed the gate. Strip the escapes with a sed that embeds the ESC via a bash $'…' literal (BSD and GNU sed both accept it) before matching. Verified end to end: with a planted unused variable the old recipe exited 0; after the fix it exits 1 and shows the warning, with and without colors. --- justfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/justfile b/justfile index 84f5090..c6ca887 100644 --- a/justfile +++ b/justfile @@ -102,8 +102,13 @@ warnings: printf '%s\n' "$out" echo; echo "cargo build failed — fix the errors above"; exit 1 fi - if printf '%s\n' "$out" | grep -qE '^warning: (unused|function|variable|field|method|struct|enum|never|dead|irrefutable|unreachable|value assigned|deprecated|trait|type|constant|static|associated)'; then - printf '%s\n' "$out" | grep -E '^warning' -A4 | grep -vE '^warning: `walgit-[a-z]+`' + # CI sets CARGO_TERM_COLOR=always, which prefixes every diagnostic with ANSI + # escapes — an anchored `^warning:` then never matches and this gate passes on + # a warning-bearing tree (issue #29). Strip the escapes before matching; the + # ESC is embedded as a bash $'…' literal so BSD and GNU sed both take it. + plain="$(printf '%s\n' "$out" | sed $'s/\x1b\\[[0-9;]*m//g')" + if printf '%s\n' "$plain" | grep -qE '^warning: (unused|function|variable|field|method|struct|enum|never|dead|irrefutable|unreachable|value assigned|deprecated|trait|type|constant|static|associated)'; then + printf '%s\n' "$plain" | grep -E '^warning' -A4 | grep -vE '^warning: `walgit-[a-z]+`' echo; echo "rustc warnings present — fix them (just warnings is part of just ci and the deploy preflight)"; exit 1 fi echo "no rustc warnings" From 2ba8318a584cc87f7063e0e93449aa857cf96f6f Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Thu, 27 Aug 2026 00:16:00 -0400 Subject: [PATCH 12/28] Run the events, follow and policy suites in the fast tier The third command of the test recipe, justfile line 86, named eight walgit-server test binaries, and crates/walgit-server/tests/ holds three more. No other recipe named them: ci at line 118 is warnings clippy test e2e, and e2e at line 90 runs only --test e2e, so events.rs, follow.rs and policy.rs never ran on a laptop or in CI even though they compile under just clippy. This appends --test events --test follow --test policy to that command. The three suites pass today, seven tests in about five seconds, and .github/workflows/ci.yml line 58 runs just test, so the change reaches CI without editing the workflow. Co-Authored-By: Claude Fable 5 --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index c6ca887..67ae5bb 100644 --- a/justfile +++ b/justfile @@ -83,7 +83,7 @@ dev-store-stop: test: {{t5}} cargo test --workspace --lib --bins {{t5}} cargo test -p walgit-store -p walgit-git -p walgit-wal -p walgit-bundle --tests - {{t5}} cargo test -p walgit-server --test web_api --test web_ui --test api_v1 --test static_http --test maintain --test routing_prefix --test lfs_upstream --test drain + {{t5}} cargo test -p walgit-server --test web_api --test web_ui --test api_v1 --test static_http --test maintain --test routing_prefix --test lfs_upstream --test drain --test events --test follow --test policy # Smart-HTTP end-to-end against real git (≈ 20 s) — run when touching smart.rs/receive/upload-pack/wal. e2e *ARGS: From 072a96bacb0fd4264f047cd1cc9f01f950d08f72 Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Thu, 27 Aug 2026 00:16:05 -0400 Subject: [PATCH 13/28] Build the web UI on Node 24 in CI .github/workflows/ci.yml pinned node-version: 22 at lines 46 and 78, while every other place that builds the SPA asks for Node 24: README.md line 118, flake.nix line 69 and Containerfile line 17. CI ran just web-build on a different major than a contributor's laptop or the OCI image, so a problem that shows up on only one of them could pass unnoticed. Both setup-node steps now say 24. GitHub Actions does not run locally here, so the check is that the workflow now agrees with those three files rather than a test. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8938635..b58ed6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: version: 10 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 24 cache: pnpm cache-dependency-path: web/pnpm-lock.yaml @@ -75,7 +75,7 @@ jobs: version: 10 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 24 cache: pnpm cache-dependency-path: web/pnpm-lock.yaml - name: Build the SPA and the SDK From a412a3b08146e25d1821807ead44dd88b24fbd6c Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Thu, 27 Aug 2026 00:16:08 -0400 Subject: [PATCH 14/28] List just clippy in the README developing block README.md line 158 called just ci "all of the above" after a list of three commands, but the recipe at justfile line 118 is ci: warnings clippy test e2e and AGENTS.md section 5 names those same four gates. A reader following the README skipped the clippy gate on a laptop and met it for the first time in CI. The block now lists just clippy next to the other three and says what just ci actually runs. This is documentation, so the check is that the list matches justfile line 118. Co-Authored-By: Claude Fable 5 --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5008ca5..e324c47 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,8 @@ and turns on `transfer.bundleURI`. `?repo=owner/name` clones right after. just test # fast hermetic tier (< 1 min): unit + quick integration, in-memory store, real git just e2e # real git against the server (~20 s) just warnings # zero rustc warnings across all targets -just ci # all of the above +just clippy # the [workspace.lints] set across all targets, warnings are errors +just ci # warnings, clippy, test, e2e: everything that must be green before a merge cargo test -p walgit-server --test sim # fault-injection simulation (crashes, partitions, stale reads) just test-s3 # store contract against local rustfs ``` From db28a8bbb6e21447e5b189a1495ef380f0bee636 Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Thu, 27 Aug 2026 00:16:54 -0400 Subject: [PATCH 15/28] Guard the dev-store podman socket bootstrap to Linux just dev-store ran the rootless podman socket bootstrap on every platform, but justfile:53 fell back to /run/user/$(id -u) when XDG_RUNTIME_DIR is unset and justfile:57 called setsid, and neither exists on macOS, so the recipe died before podman compose up. The bootstrap now runs only when uname -s reports Linux, and setsid is dropped because nohup with a redirect already detaches the service. On any other system the recipe makes one podman info probe and, when that probe fails, prints a single line asking the user to start the container runtime first (podman machine start on macOS) before exiting 1. Proved by extracting the recipe body to a file and running bash -n on it, then running that body on this macOS host, where it printed the one line and exited 1 without reaching podman compose. Co-Authored-By: Claude Fable 5 --- justfile | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/justfile b/justfile index 67ae5bb..57eca9d 100644 --- a/justfile +++ b/justfile @@ -41,22 +41,31 @@ dev-local config="walgit.standalone.toml": # Start rustfs (S3-compatible) for local dev via podman compose (rootless, no daemon group needed; # `podman compose` drives compose.yaml through the docker-compose binary dev.yml installs). -# `podman compose` talks to the podman API socket; rootless nix podman has no systemd unit for it, so -# `podman system service` is started (detached, idle-timeout 0) when the socket is missing. +# `podman compose` talks to the podman API socket; on Linux rootless nix podman has no systemd unit +# for it, so `podman system service` is started (detached, idle-timeout 0) when the socket is missing. +# Elsewhere (macOS, the BSDs) the socket belongs to the podman machine VM: the recipe only checks that +# podman answers and tells you to start it if it does not. dev-store: #!/usr/bin/env bash set -euo pipefail - # nix podman ships no /etc/containers: give the user a signature policy + registry search list once. - cdir="${XDG_CONFIG_HOME:-$HOME/.config}/containers"; mkdir -p "$cdir" - [ -f "$cdir/policy.json" ] || printf '{"default":[{"type":"insecureAcceptAnything"}]}\n' > "$cdir/policy.json" - [ -f "$cdir/registries.conf" ] || printf 'unqualified-search-registries = ["docker.io"]\n' > "$cdir/registries.conf" - sock="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/podman/podman.sock" - if [ ! -S "$sock" ]; then - echo "starting rootless podman API socket at $sock" - mkdir -p "$(dirname "$sock")" - setsid nohup podman system service --time=0 "unix://$sock" >/tmp/walgit-podman-service.log 2>&1 < /dev/null & - for _ in $(seq 1 50); do [ -S "$sock" ] && break; sleep 0.2; done - [ -S "$sock" ] || { echo "podman API socket did not appear; see /tmp/walgit-podman-service.log"; exit 1; } + # The rootless socket bootstrap is Linux-only: XDG_RUNTIME_DIR and /run/user do not exist on + # macOS or the BSDs, and setsid is util-linux. There the socket lives in the podman machine VM. + if [ "$(uname -s)" = Linux ]; then + # nix podman ships no /etc/containers: give the user a signature policy + registry search list once. + cdir="${XDG_CONFIG_HOME:-$HOME/.config}/containers"; mkdir -p "$cdir" + [ -f "$cdir/policy.json" ] || printf '{"default":[{"type":"insecureAcceptAnything"}]}\n' > "$cdir/policy.json" + [ -f "$cdir/registries.conf" ] || printf 'unqualified-search-registries = ["docker.io"]\n' > "$cdir/registries.conf" + sock="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/podman/podman.sock" + if [ ! -S "$sock" ]; then + echo "starting rootless podman API socket at $sock" + mkdir -p "$(dirname "$sock")" + nohup podman system service --time=0 "unix://$sock" >/tmp/walgit-podman-service.log 2>&1 < /dev/null & + for _ in $(seq 1 50); do [ -S "$sock" ] && break; sleep 0.2; done + [ -S "$sock" ] || { echo "podman API socket did not appear; see /tmp/walgit-podman-service.log"; exit 1; } + fi + elif ! podman info >/dev/null 2>&1; then + echo "podman is not answering: start the container runtime first (macOS: podman machine start)" + exit 1 fi podman compose up -d rustfs echo "Waiting for rustfs to be healthy..." From f509b50847208cd3c55bf3a3986ddbfaaae48a5b Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Thu, 27 Aug 2026 00:21:23 -0400 Subject: [PATCH 16/28] Guard the dev-local SPA build on web/dist/repos.js justfile:33 skipped the SPA build whenever web/dist/index.html existed, but crates/walgit-server/build.rs:19-21 drops a placeholder index.html into web/dist on any cargo build of a fresh clone, so after the first cargo run just dev-local never built the real UI and served that placeholder page instead. The guard now tests web/dist/repos.js, which only a real Vite build produces and which Containerfile:23 already treats as the proof of one, and the message says the SPA is unbuilt rather than missing. Proved by moving the real web/dist aside and running the compiled walgit-server build script directly, which wrote the 220 byte placeholder index.html; against that tree the old condition skipped the build and the new one runs it, while against the restored Vite output the new condition stays quiet. The recipe body also passes bash -n after extraction, with and without just's {{config}} interpolation expanded. Co-Authored-By: Claude Fable 5 --- justfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/justfile b/justfile index 57eca9d..c852b66 100644 --- a/justfile +++ b/justfile @@ -18,7 +18,7 @@ web-build: # Local dev = standalone: the server with every role (serve, maintain, events) at # https://walgit.localhost:$PORT (default 8080) against local rustfs. Self-contained: starts rustfs (+ bucket) if -# it is not answering on :9000 and builds the SPA if web/dist is missing, then runs the server. +# it is not answering on :9000 and builds the SPA if web/dist holds no Vite output, then runs the server. # `config` defaults to walgit.standalone.toml; point it at a real bucket by editing [store] there. The rustfs # keys come from the environment (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY; compose.yaml fixes them). # Optional: export WALGIT__SERVER__AUTH__* (OIDC client, session secret) to try browser sign-in locally. @@ -30,8 +30,10 @@ dev-local config="walgit.standalone.toml": echo "rustfs not running on :9000 — starting it (just dev-store)" just dev-store fi - if [ ! -f web/dist/index.html ]; then - echo "web/dist missing — building the SPA (just web-build)" + # crates/walgit-server/build.rs:19 writes a placeholder index.html on any cargo build, so only + # repos.js proves a real Vite build (the same file Containerfile:23 checks). + if [ ! -f web/dist/repos.js ]; then + echo "web/dist SPA is unbuilt; building it (just web-build)" just web-build fi cargo build --release --bin walgit-server From 196458331c1ede3c630a4836394cc759e3148152 Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Thu, 27 Aug 2026 00:26:11 -0400 Subject: [PATCH 17/28] Survive empty array expansion in e2e.sh under bash 3.2 tests/e2e.sh runs under set -euo pipefail, and bash 3.2, the /bin/bash macOS ships, calls the expansion of an empty array an unbound variable. The EXIT trap at line 73 therefore aborted on the PIDS expansion before rm -rf "$TMP" and leaked the temp directory, and the same expansion at lines 49, 51 and 56 stopped the run at its first health check. PIDS now takes the :- guard that tests/git-bundle-filter.sh:24 already uses, while the three argument lists take the +alternate form instead, because the :- form hands curl and git a blank argument they reject with "option : blank argument where content is expected". Proved on this macOS host with bash 3.2.57 by running tests/e2e.sh with WALGIT_E2E_BASE_URL pointed at a memory-backend server on loopback: before the change it died at line 56 with "AUTH_CURL_ARGS[@]: unbound variable", after it reaches the synth step and its mktemp directory is gone once the script exits. Co-Authored-By: Claude Fable 5 --- tests/e2e.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/e2e.sh b/tests/e2e.sh index 82be1d9..cc5bba7 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -45,15 +45,17 @@ if [[ -n "${WALGIT_E2E_BASE_URL:-}" ]]; then walgit_auth_setup "$WALGIT_E2E_BASE_URL" || exit 1 fi +# ${ARR[@]+"${ARR[@]}"}: bash 3.2 (macOS /bin/bash) calls an empty array expansion an unbound +# variable under set -u. "${ARR[@]:-}" would pass a blank argument, which curl and git reject. # Wrapper for curl that adds auth headers. -curl_auth() { curl "${AUTH_CURL_ARGS[@]}" "$@"; } +curl_auth() { curl ${AUTH_CURL_ARGS[@]+"${AUTH_CURL_ARGS[@]}"} "$@"; } # Wrapper for git that adds auth headers. -git_auth() { git "${GIT_AUTH_ARGS[@]}" "$@"; } +git_auth() { git ${GIT_AUTH_ARGS[@]+"${GIT_AUTH_ARGS[@]}"} "$@"; } wait_http() { local url="$1" max="${2:-30}" for ((i=0; i/dev/null 2>&1; then return 0; fi + if curl -sf ${AUTH_CURL_ARGS[@]+"${AUTH_CURL_ARGS[@]}"} "$url" >/dev/null 2>&1; then return 0; fi sleep 1 done return 1 @@ -70,7 +72,9 @@ fi TMP="$(mktemp -d)" PIDS=() -cleanup() { for p in "${PIDS[@]}"; do kill "$p" 2>/dev/null || true; done; wait 2>/dev/null || true; rm -rf "$TMP"; } +# "${PIDS[@]:-}": bash 3.2 (macOS /bin/bash) calls an empty array expansion an unbound variable +# under set -u, which aborts the EXIT trap before rm -rf. Same guard as tests/git-bundle-filter.sh:24. +cleanup() { for p in "${PIDS[@]:-}"; do kill "$p" 2>/dev/null || true; done; wait 2>/dev/null || true; rm -rf "$TMP"; } trap cleanup EXIT PORT="$(rand_port)" From 9f53c7dac79fcf79f3757b6fcf802b3d7a1c4db9 Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Thu, 27 Aug 2026 00:27:17 -0400 Subject: [PATCH 18/28] Stop e2e synth from reading a local-mode-only config tests/e2e.sh line 126 passed --config "$TMP/walgit.toml" to synth, but that file is written only by the heredoc at lines 92 to 108 inside the local-server branch, so a run with WALGIT_E2E_BASE_URL set died on its first command with "config file ... not found" and exit 2. synth reads nothing out of the config, since crates/walgit-cli/src/lib.rs:493 dispatches it with out, size, commits, files and seed alone, so the call now passes --config /dev/null, which crates/walgit-cli/src/lib.rs:451-455 names as the way to ask for defaults on purpose. That is one token against moving a seventeen line heredoc, and it also keeps remote mode from writing a config whose listen address and cache directory no server in that mode ever reads. Proved by running tests/e2e.sh with WALGIT_E2E_BASE_URL pointed at a memory-backend walgit on loopback, where it now passes every step from synth through DELETE repo, and by running the script in local mode, which still passes. Co-Authored-By: Claude Fable 5 --- tests/e2e.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e.sh b/tests/e2e.sh index cc5bba7..a36fa8e 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -127,7 +127,9 @@ fi step "synth: generate synthetic repo (size s, seed 12345)" SYNTH_DIR="$TMP/synth" -"$WALGIT" --config "$TMP/walgit.toml" synth --out "$SYNTH_DIR" --size s --seed 12345 +# synth reads nothing from the config (walgit-cli/src/lib.rs:493 passes only out/size/seed) and +# $TMP/walgit.toml exists in local mode only, so ask for defaults the way the CLI documents. +"$WALGIT" --config /dev/null synth --out "$SYNTH_DIR" --size s --seed 12345 pass "synth completed" # Verify with git fsck. From 924f0195d830f56afa82e2dd5eb6af4d6b2802c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20L=C3=BCtke?= Date: Sat, 5 Sep 2026 19:29:19 +0000 Subject: [PATCH 19/28] Fix S3 retry classification after the #18 merge classify_put_error/classify_list_error still took references from main while classify_error and the new tests took ownership, so the crate did not compile. Take ownership throughout so a SlowDown on the manifest CAS is Retryable on S3 the way it already is on GCS. --- crates/walgit-store/src/s3.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/walgit-store/src/s3.rs b/crates/walgit-store/src/s3.rs index 5670db8..ffa04a7 100644 --- a/crates/walgit-store/src/s3.rs +++ b/crates/walgit-store/src/s3.rs @@ -306,9 +306,9 @@ where fn classify_put_error( key: &str, - err: &aws_sdk_s3::error::SdkError, + err: aws_sdk_s3::error::SdkError, ) -> StoreError { - let code = err_code(err).unwrap_or(""); + let code = err_code(&err).unwrap_or(""); match code { "PreconditionFailed" | "ConditionalRequestConflict" => StoreError::PreconditionFailed { key: key.into(), @@ -319,7 +319,7 @@ fn classify_put_error( } fn classify_list_error( - err: &aws_sdk_s3::error::SdkError, + err: aws_sdk_s3::error::SdkError, ) -> StoreError { classify_error("s3 list error", err) } @@ -412,7 +412,7 @@ impl ObjectStore for S3Store { }) } Err(e) => { - let mut err = classify_put_error(key, &e); + let mut err = classify_put_error(key, e); // Fill `current` via HEAD if we got a PreconditionFailed. if let StoreError::PreconditionFailed { current: c, .. } = &mut err && c.is_none() @@ -545,7 +545,7 @@ impl ObjectStore for S3Store { let item = state.buffer.next(); item.map(|i| (i, state)) } - Err(err) => Some((Err(classify_list_error(&err)), state)), + Err(err) => Some((Err(classify_list_error(err)), state)), } }, )) @@ -565,7 +565,7 @@ impl ObjectStore for S3Store { if let Some(ct) = &continuation_token { builder = builder.continuation_token(ct); } - let resp = builder.send().await.map_err(|e| classify_list_error(&e))?; + let resp = builder.send().await.map_err(classify_list_error)?; out.extend( resp.common_prefixes() .iter() From 2c5af1a1ad55012dbbd10e0ef34982fb9536c6c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20L=C3=BCtke?= Date: Sat, 5 Sep 2026 20:07:52 +0000 Subject: [PATCH 20/28] Make the #18/#25 merges clippy-clean classify_error took SdkError by value and never consumed it (clippy::needless_pass_by_value). Take a reference throughout. The new api_v1 admin-gate test declared consts after statements. --- crates/walgit-server/tests/api_v1.rs | 4 +-- crates/walgit-store/src/s3.rs | 52 ++++++++++++++-------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/crates/walgit-server/tests/api_v1.rs b/crates/walgit-server/tests/api_v1.rs index e4cc24f..89f2f0e 100644 --- a/crates/walgit-server/tests/api_v1.rs +++ b/crates/walgit-server/tests/api_v1.rs @@ -564,6 +564,8 @@ async fn repository_delete_requires_admin() -> TestResult { /// but the policy and settings documents move only with admin. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn policy_and_settings_writes_require_admin() -> TestResult { + const POLICY: &str = r#"{"version":1,"groups":[],"rules":[]}"#; + const SETTINGS: &str = "[bundles]\nmin_commits = 3\n"; let server = Server::start_with_tweak(|c| { c.server.auth.mode = walgit_config::AuthMode::Token; c.server.auth.anonymous_read = false; @@ -595,8 +597,6 @@ async fn policy_and_settings_writes_require_admin() -> TestResult { "write permission creates the repository" ); - const POLICY: &str = r#"{"version":1,"groups":[],"rules":[]}"#; - const SETTINGS: &str = "[bundles]\nmin_commits = 3\n"; for (path, body) in [ ("/gates/repo/api/policy", POLICY), ("/gates/repo/api/settings", SETTINGS), diff --git a/crates/walgit-store/src/s3.rs b/crates/walgit-store/src/s3.rs index ffa04a7..d788cc2 100644 --- a/crates/walgit-store/src/s3.rs +++ b/crates/walgit-store/src/s3.rs @@ -293,11 +293,11 @@ where /// Wrap an SDK failure, keeping the retryable/permanent distinction that /// `StoreError::is_retryable` is read for. -fn classify_error(context: &str, err: aws_sdk_s3::error::SdkError) -> StoreError +fn classify_error(context: &str, err: &aws_sdk_s3::error::SdkError) -> StoreError where E: aws_sdk_s3::error::ProvideErrorMetadata + std::error::Error + Send + Sync + 'static, { - if is_retryable(&err) { + if is_retryable(err) { StoreError::Retryable(anyhow::anyhow!("{context}: {err}")) } else { StoreError::Other(anyhow::anyhow!("{context}: {err}")) @@ -306,9 +306,9 @@ where fn classify_put_error( key: &str, - err: aws_sdk_s3::error::SdkError, + err: &aws_sdk_s3::error::SdkError, ) -> StoreError { - let code = err_code(&err).unwrap_or(""); + let code = err_code(err).unwrap_or(""); match code { "PreconditionFailed" | "ConditionalRequestConflict" => StoreError::PreconditionFailed { key: key.into(), @@ -319,7 +319,7 @@ fn classify_put_error( } fn classify_list_error( - err: aws_sdk_s3::error::SdkError, + err: &aws_sdk_s3::error::SdkError, ) -> StoreError { classify_error("s3 list error", err) } @@ -361,7 +361,7 @@ impl ObjectStore for S3Store { { return Ok(None); } - Err(classify_error("s3 head error", err)) + Err(classify_error("s3 head error", &err)) } } } @@ -412,7 +412,7 @@ impl ObjectStore for S3Store { }) } Err(e) => { - let mut err = classify_put_error(key, e); + let mut err = classify_put_error(key, &e); // Fill `current` via HEAD if we got a PreconditionFailed. if let StoreError::PreconditionFailed { current: c, .. } = &mut err && c.is_none() @@ -466,7 +466,7 @@ impl ObjectStore for S3Store { return Ok(()); } } - Err(classify_error("s3 delete error", err)) + Err(classify_error("s3 delete error", &err)) } } } @@ -545,7 +545,7 @@ impl ObjectStore for S3Store { let item = state.buffer.next(); item.map(|i| (i, state)) } - Err(err) => Some((Err(classify_list_error(err)), state)), + Err(err) => Some((Err(classify_list_error(&err)), state)), } }, )) @@ -565,7 +565,7 @@ impl ObjectStore for S3Store { if let Some(ct) = &continuation_token { builder = builder.continuation_token(ct); } - let resp = builder.send().await.map_err(classify_list_error)?; + let resp = builder.send().await.map_err(|e| classify_list_error(&e))?; out.extend( resp.common_prefixes() .iter() @@ -666,7 +666,7 @@ impl ObjectStore for S3Store { let upload = create .send() .await - .map_err(|e| classify_error("s3 create multipart", e))?; + .map_err(|e| classify_error("s3 create multipart", &e))?; let upload_id = upload .upload_id() .ok_or_else(|| { @@ -709,7 +709,7 @@ impl ObjectStore for S3Store { .copy_source_range(format!("bytes={from}-{}", from + len - 1)) .send() .await - .map_err(|e| classify_error("s3 upload part copy", e))?; + .map_err(|e| classify_error("s3 upload part copy", &e))?; let etag = part .copy_part_result() .and_then(|r| r.e_tag()) @@ -761,7 +761,7 @@ impl ObjectStore for S3Store { .content_length(i64::try_from(len).map_err(StoreError::other)?) .send() .await - .map_err(|e| classify_error("s3 upload part", e))?; + .map_err(|e| classify_error("s3 upload part", &e))?; parts.push( aws_sdk_s3::types::CompletedPart::builder() .e_tag(part.e_tag().unwrap_or("").to_owned()) @@ -795,7 +795,7 @@ impl ObjectStore for S3Store { Ok(r) => r, Err(e) => { let _ = self.abort_multipart(dest, &upload_id).await; - return Err(classify_error("s3 complete multipart", e)); + return Err(classify_error("s3 complete multipart", &e)); } }; let etag = resp.e_tag().map(|s| s.trim_matches('"').to_owned()); @@ -857,7 +857,7 @@ impl S3Store { let upload = create .send() .await - .map_err(|e| classify_error("s3 create multipart", e))?; + .map_err(|e| classify_error("s3 create multipart", &e))?; let upload_id = upload .upload_id() @@ -919,7 +919,7 @@ impl S3Store { Ok(p) => p, Err(e) => { let _ = self.abort_multipart(key, &upload_id).await; - return Err(classify_error("s3 upload part", e)); + return Err(classify_error("s3 upload part", &e)); } }; @@ -952,7 +952,7 @@ impl S3Store { Ok(r) => r, Err(e) => { let _ = self.abort_multipart(key, &upload_id).await; - return Err(classify_error("s3 complete multipart", e)); + return Err(classify_error("s3 complete multipart", &e)); } }; @@ -972,7 +972,7 @@ impl S3Store { .upload_id(upload_id) .send() .await - .map_err(|e| classify_error("abort multipart", e))?; + .map_err(|e| classify_error("abort multipart", &e))?; Ok(()) } } @@ -1072,7 +1072,7 @@ mod tests { async fn throttling_is_retryable() { let client = fake_s3(503, "SlowDown").await; assert!(matches!( - classify_put_error("k", put_error(&client).await), + classify_put_error("k", &put_error(&client).await), StoreError::Retryable(_) )); } @@ -1081,7 +1081,7 @@ mod tests { async fn server_fault_is_retryable() { let client = fake_s3(500, "InternalError").await; assert!(matches!( - classify_put_error("k", put_error(&client).await), + classify_put_error("k", &put_error(&client).await), StoreError::Retryable(_) )); } @@ -1090,7 +1090,7 @@ mod tests { async fn a_transient_status_without_a_known_code_is_retryable() { let client = fake_s3(504, "SomethingUnrecognised").await; assert!(matches!( - classify_put_error("k", put_error(&client).await), + classify_put_error("k", &put_error(&client).await), StoreError::Retryable(_) )); } @@ -1100,7 +1100,7 @@ mod tests { // Nothing listens on port 1: a dispatch failure, no response at all. let client = client_for("http://127.0.0.1:1"); assert!(matches!( - classify_put_error("k", put_error(&client).await), + classify_put_error("k", &put_error(&client).await), StoreError::Retryable(_) )); } @@ -1109,7 +1109,7 @@ mod tests { async fn denied_is_permanent() { let client = fake_s3(403, "AccessDenied").await; assert!(matches!( - classify_put_error("k", put_error(&client).await), + classify_put_error("k", &put_error(&client).await), StoreError::Other(_) )); } @@ -1118,7 +1118,7 @@ mod tests { async fn a_failed_precondition_stays_a_failed_precondition() { let client = fake_s3(412, "PreconditionFailed").await; assert!(matches!( - classify_put_error("k", put_error(&client).await), + classify_put_error("k", &put_error(&client).await), StoreError::PreconditionFailed { .. } )); } @@ -1127,7 +1127,7 @@ mod tests { async fn a_throttled_list_is_retryable() { let client = fake_s3(503, "SlowDown").await; assert!(matches!( - classify_list_error(list_error(&client).await), + classify_list_error(&list_error(&client).await), StoreError::Retryable(_) )); } @@ -1136,7 +1136,7 @@ mod tests { async fn a_denied_list_is_permanent() { let client = fake_s3(403, "AccessDenied").await; assert!(matches!( - classify_list_error(list_error(&client).await), + classify_list_error(&list_error(&client).await), StoreError::Other(_) )); } From e1589bcf6d927138278586f9e7743a6fb11db760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20L=C3=BCtke?= Date: Sat, 5 Sep 2026 20:20:37 +0000 Subject: [PATCH 21/28] Use assert_eq! in the bundle verify test CI pins rustc 1.97.1; clippy::manual_assert_eq is new there and `just clippy` is -D warnings. --- crates/walgit-bundle/tests/bundle.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/walgit-bundle/tests/bundle.rs b/crates/walgit-bundle/tests/bundle.rs index 97dc328..e70e042 100644 --- a/crates/walgit-bundle/tests/bundle.rs +++ b/crates/walgit-bundle/tests/bundle.rs @@ -337,7 +337,7 @@ async fn full_bundle_passes_verify() { assert!(!entry.tips.is_empty(), "bundle entry should have tips"); assert!(entry.tips.iter().any(|t| t.name == "refs/heads/main")); assert!(entry.tips.iter().any(|t| t.name == "refs/tags/v1.0")); - assert!(entry.kind == "full"); + assert_eq!(entry.kind, "full"); assert!(entry.base_id.is_empty()); } From a9da38f32b6e8b45bf8c22de7441f7db920854fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20L=C3=BCtke?= Date: Sat, 5 Sep 2026 20:37:44 +0000 Subject: [PATCH 22/28] Stop reading past the git bundle header in tests `incremental_has_prerequisites` took the first 20 lossy-UTF-8 lines of the bundle file, which walks into PACK bytes. A binary line that started with '-' was then compared to the base tips and failed CI. --- crates/walgit-bundle/tests/bundle.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/walgit-bundle/tests/bundle.rs b/crates/walgit-bundle/tests/bundle.rs index e70e042..c0118c0 100644 --- a/crates/walgit-bundle/tests/bundle.rs +++ b/crates/walgit-bundle/tests/bundle.rs @@ -390,20 +390,22 @@ async fn incremental_has_prerequisites() { String::from_utf8_lossy(&output.stderr) ); - // Check that the bundle header has prerequisites (lines starting with -). + // Bundle header ends at the first blank line; the pack is binary after that. let header = String::from_utf8_lossy(&data); - let header_lines: Vec<&str> = header.lines().take(20).collect(); - let has_prereq = header_lines.iter().any(|l| l.starts_with('-')); + let header_lines: Vec<&str> = header.lines().take_while(|l| !l.is_empty()).collect(); + let prereqs: Vec<&str> = header_lines + .iter() + .filter_map(|l| l.strip_prefix('-')) + .filter_map(|rest| rest.split_whitespace().next()) + .collect(); assert!( - has_prereq, + !prereqs.is_empty(), "incremental bundle should have prerequisites in header" ); // The prerequisites should match the base bundle's tips. let base_tips: Vec<&str> = base_entry.tips.iter().map(|t| t.oid.as_str()).collect(); - for prereq_line in header_lines.iter().filter(|l| l.starts_with('-')) { - // Format: "- " - let oid = prereq_line[1..].split_whitespace().next().unwrap_or(""); + for oid in prereqs { assert!( base_tips.contains(&oid), "prerequisite {oid} should be in base tips {base_tips:?}" From e5295e6ee45f5267c661f8bbd27ed0a07e55e7db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20L=C3=BCtke?= Date: Sat, 5 Sep 2026 20:55:42 +0000 Subject: [PATCH 23/28] Give the integration cargo test invocations 10 minutes CI's `just test` second command spent ~4.5 minutes compiling test binaries inside `timeout 300`, then SIGTERM'd wal.rs mid-run (exit 124). The suites themselves finish in seconds once built. --- justfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/justfile b/justfile index c852b66..e4a76f4 100644 --- a/justfile +++ b/justfile @@ -93,8 +93,8 @@ dev-store-stop: # hung test blocks for the whole timeout. Use `just e2e` / `just ci` below. test: {{t5}} cargo test --workspace --lib --bins - {{t5}} cargo test -p walgit-store -p walgit-git -p walgit-wal -p walgit-bundle --tests - {{t5}} cargo test -p walgit-server --test web_api --test web_ui --test api_v1 --test static_http --test maintain --test routing_prefix --test lfs_upstream --test drain --test events --test follow --test policy + {{t10}} cargo test -p walgit-store -p walgit-git -p walgit-wal -p walgit-bundle --tests + {{t10}} cargo test -p walgit-server --test web_api --test web_ui --test api_v1 --test static_http --test maintain --test routing_prefix --test lfs_upstream --test drain --test events --test follow --test policy # Smart-HTTP end-to-end against real git (≈ 20 s) — run when touching smart.rs/receive/upload-pack/wal. e2e *ARGS: From d5e75caf53ee4b0d8690b460efd33126be0ef0ad Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Sun, 6 Sep 2026 13:18:55 -0400 Subject: [PATCH 24/28] Check pushed tips even when the pack is empty A ref-only push carries a 32-byte pack with zero objects, and receive-pack skipped the connectivity check whenever ingest returned Ok(None). A command line naming an object the server does not have was therefore accepted, the ref was published to the WAL, and every clone that walked it failed with "missing object" (#37, reported with a repro by czk-aa). The check now runs whenever unpack succeeded and any update names a non-zero tip. With wal.check_connectivity the existing walk covers the tips and everything new under them and stops at existing refs, so a legitimate ref-only push costs a few lookups; with it turned off the tips are still looked up one by one before anything is published. The e2e test posts the exact empty pack from the report against both settings and expects ng refs/heads/ghost, then pushes main to a new branch, which sends the same zero-object pack, and expects it to land. The test fails on main at the ng assertion and passes here. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01CSbhRh6UEjFrgBYYzucvDe --- crates/walgit-server/src/smart.rs | 53 ++++++++++++++++--------- crates/walgit-server/tests/e2e.rs | 64 +++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 18 deletions(-) diff --git a/crates/walgit-server/src/smart.rs b/crates/walgit-server/src/smart.rs index da59a86..4659606 100644 --- a/crates/walgit-server/src/smart.rs +++ b/crates/walgit-server/src/smart.rs @@ -1178,31 +1178,48 @@ async fn receive_pack_process( Err(e) => Some(format!("unpack failed: {e}")), }; - // Connectivity check for pushed tips (before we publish anything). - if unpack_err.is_none() - && st.cfg.wal.check_connectivity - && let Ok(Some(_)) = &ingest - { + // Every pushed tip must exist before anything is published, pack or no + // pack: a ref-only push carries a zero-object pack (`ingest` is `Ok(None)`) + // and this block used to be skipped for it, so a ref could be published + // pointing at an object nobody has (#37). With `wal.check_connectivity` + // the walk covers the tips and everything new under them; without it the + // tips themselves are still looked up. + if unpack_err.is_none() { let tips: Vec = txn .updates .iter() .filter(|u| !u.new_oid.is_empty() && !is_zero_oid(&u.new_oid)) .filter_map(|u| gix_hash::ObjectId::from_hex(u.new_oid.as_bytes()).ok()) .collect(); - if !tips.is_empty() - && let Err(e) = local - .check_connectivity_async(&tips, true) - .instrument(tracing::info_span!( - "receive.connectivity", - tips = tips.len() - )) + if !tips.is_empty() { + let verdict: Result<(), String> = if st.cfg.wal.check_connectivity { + local + .check_connectivity_async(&tips, true) + .instrument(tracing::info_span!( + "receive.connectivity", + tips = tips.len() + )) + .await + .map_err(|e| format!("connectivity: {e}")) + } else { + let repo = local.clone(); + let tips = tips.clone(); + tokio::task::spawn_blocking(move || { + tips.iter() + .find(|t| !repo.has_object(t)) + .map_or(Ok(()), |t| Err(format!("missing object {t}"))) + }) .await - { - // Every refusal names the reason on each ref: `unpack ng` - // alone makes git print "remote failed to report status". - tracing::warn!(repo = %route_id, error = %e, "receive-pack: connectivity check failed"); - metrics::counter!("walgit_push_refused_total", "reason" => "connectivity").increment(1); - return Ok(refusal_report(&caps, &txn, &format!("connectivity: {e}")).await); + .map_err(|e| ApiError::Internal(format!("tip check: {e}")))? + }; + if let Err(msg) = verdict { + // Every refusal names the reason on each ref: `unpack ng` + // alone makes git print "remote failed to report status". + tracing::warn!(repo = %route_id, error = %msg, "receive-pack: tip check failed"); + metrics::counter!("walgit_push_refused_total", "reason" => "connectivity") + .increment(1); + return Ok(refusal_report(&caps, &txn, &msg).await); + } } } diff --git a/crates/walgit-server/tests/e2e.rs b/crates/walgit-server/tests/e2e.rs index 7d165be..7c67cb8 100644 --- a/crates/walgit-server/tests/e2e.rs +++ b/crates/walgit-server/tests/e2e.rs @@ -3058,3 +3058,67 @@ async fn reads_after_an_acknowledged_push_never_show_the_previous_tip() -> TestR assert!(stale.is_empty(), "stale reads:\n{}", stale.join("\n")); Ok(()) } + +/// #37: a ref-only push carries a 32-byte zero-object pack, and receive-pack used to skip the +/// connectivity check for it, so `refs/heads/ghost` could be published pointing at an object +/// nobody has, after which every clone walking it died with `missing object`. The tip is now +/// checked like any other, and a ref-only push to an object the server does have still lands. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn empty_pack_push_to_a_missing_object_is_refused() -> TestResult { + for check_connectivity in [true, false] { + empty_pack_push_is_refused_with(check_connectivity).await?; + } + Ok(()) +} + +/// Both tip checks refuse it: the full walk, and the bare lookup a host with +/// `wal.check_connectivity = false` falls back to. +async fn empty_pack_push_is_refused_with(check_connectivity: bool) -> TestResult { + let server = + Server::start_with_tweak(|c| c.wal.check_connectivity = check_connectivity).await?; + server.put_repo("t", "ghost").await?; + let src = TestRepo::synthetic(2, 2)?; + git_in(&src, &["branch", "-M", "main"])?; + git_in( + &src, + &["remote", "add", "origin", &server.repo_url("t", "ghost")], + )?; + git_in(&src, &["push", "-q", "origin", "main"])?; + + // One command line, a flush, then the empty pack: header, zero objects, its checksum. + let cmd = format!( + "{} {} refs/heads/ghost\0report-status\n", + "0".repeat(40), + "b".repeat(40) + ); + let mut body = format!("{:04x}{cmd}0000", cmd.len() + 4).into_bytes(); + body.extend_from_slice(b"PACK\x00\x00\x00\x02\x00\x00\x00\x00"); + body.extend_from_slice(&[ + 0x02, 0x9d, 0x08, 0x82, 0x3b, 0xd8, 0xa8, 0xea, 0xb5, 0x10, 0xad, 0x6a, 0xc7, 0x5c, 0x82, + 0x3c, 0xfd, 0x3e, 0xd3, 0x1e, + ]); + let resp = reqwest::Client::new() + .post(format!("{}/t/ghost.git/git-receive-pack", server.base_url)) + .header("Content-Type", "application/x-git-receive-pack-request") + .body(body) + .send() + .await?; + assert_eq!(resp.status(), 200); + let report = resp.text().await?; + assert!( + report.contains("ng refs/heads/ghost"), + "check_connectivity={check_connectivity}: {report}" + ); + assert!(!report.contains("ok refs/heads/ghost"), "{report}"); + let refs = git_in(&src, &["ls-remote", "origin"])?; + assert!(!refs.contains("refs/heads/ghost"), "{refs}"); + + // The legitimate shape of the same wire bytes: a new branch at an object the server has. + git_in(&src, &["push", "-q", "origin", "main:refs/heads/copy"])?; + let refs = git_in(&src, &["ls-remote", "origin"])?; + assert!( + refs.contains("refs/heads/copy"), + "check_connectivity={check_connectivity}: {refs}" + ); + Ok(()) +} From 41faf55193de91609e9965a6d435412fb81446bf Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Mon, 7 Sep 2026 12:05:15 -0400 Subject: [PATCH 25/28] Let just clippy pass on macOS The two statvfs helpers, disk_usage in walgit-wal/src/registry.rs and disk_avail in walgit-server/src/rebuild.rs, widen the block-count fields with `as u64`. Those fields are u32 on macOS and u64 on Linux, so on a Mac the strict set fails with cast_lossless and the gate #15 added never passes locally, while `u64::from` would be a useless conversion on Linux where CI runs. No spelling satisfies both platforms, so each helper carries a targeted allow with a comment saying why, the shape the Cargo.toml lint notes ask for. Checked with cargo clippy --workspace --all-targets -- -D warnings on macOS, now clean; the change is a no-op on Linux. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01CSbhRh6UEjFrgBYYzucvDe --- crates/walgit-server/src/rebuild.rs | 3 +++ crates/walgit-wal/src/registry.rs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/crates/walgit-server/src/rebuild.rs b/crates/walgit-server/src/rebuild.rs index 560951e..8247234 100644 --- a/crates/walgit-server/src/rebuild.rs +++ b/crates/walgit-server/src/rebuild.rs @@ -125,6 +125,9 @@ fn copy_tree(src: &Path, dst: &Path) -> std::io::Result { Ok(bytes) } +// statvfs's block fields are u32 on macOS and u64 on Linux, so `as u64` is the one spelling +// that is lossless on both; `From` would be a useless conversion on Linux. +#[allow(clippy::cast_lossless)] fn disk_avail(path: &Path) -> Option { use std::ffi::CString; use std::os::unix::ffi::OsStrExt; diff --git a/crates/walgit-wal/src/registry.rs b/crates/walgit-wal/src/registry.rs index 6f30f8f..5215d4c 100644 --- a/crates/walgit-wal/src/registry.rs +++ b/crates/walgit-wal/src/registry.rs @@ -478,6 +478,9 @@ fn dir_size(path: &std::path::Path) -> u64 { } /// (used, total) bytes of the filesystem holding `path` (statvfs). +// statvfs's block fields are u32 on macOS and u64 on Linux, so `as u64` is the one spelling +// that is lossless on both; `From` would be a useless conversion on Linux. +#[allow(clippy::cast_lossless)] fn disk_usage(path: &std::path::Path) -> Option<(u64, u64)> { use std::ffi::CString; use std::os::unix::ffi::OsStrExt; From 9911cb18bae2318f8002fd37220dbcaaebcb5ae9 Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Mon, 7 Sep 2026 12:14:23 -0400 Subject: [PATCH 26/28] Write faulted objects off the async worker The web object faulter read objects from a remote pack set asynchronously and then called `write_loose_object` for each of them on the same tokio worker, so one request for a commit diff could spend seconds stating paths, deflating and renaming files while it held a runtime thread. `fault_many` now collects the reads of each chunk of 32 and hands the whole chunk to one `tokio::task::spawn_blocking`, and `fault` writes its single object through the same helper. Behaviour is unchanged: the same objects land in the loose store, ids already faulted are still skipped, and the error text is still `fault object : `, with a join failure reported as an internal error. AGENTS.md principle VI says never block the async runtime, and `spawn_blocking` is the pattern the rest of the crate already uses. Checked with `cargo clippy -p walgit-server --all-targets --no-deps` (clean apart from the pre-existing `rebuild.rs:141` cast lint), the `web_api`, `web_ui` and `api_v1` suites, `cargo test -p walgit-server --test e2e -- remote`, the e2e `blocking_work_in_the_install_path_does_not_stall_requests`, and `cargo fmt --all -- --check`. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01CSbhRh6UEjFrgBYYzucvDe --- crates/walgit-server/src/web/objects.rs | 46 +++++++++++++++++++------ 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/crates/walgit-server/src/web/objects.rs b/crates/walgit-server/src/web/objects.rs index 73c819e..73e7ab1 100644 --- a/crates/walgit-server/src/web/objects.rs +++ b/crates/walgit-server/src/web/objects.rs @@ -88,7 +88,7 @@ impl Remote { /// Read + write into the local loose store (so git can see it). pub async fn fault(&self, oid: &gix_hash::oid) -> Result, ApiError> { let o = self.get(oid).await?; - self.write_local(oid, &o)?; + self.write_local(vec![(oid.to_owned(), o.clone())]).await?; Ok(o) } @@ -104,23 +104,49 @@ impl Remote { }; for chunk in todo.chunks(PAR) { let results = futures::future::join_all(chunk.iter().map(|o| self.get(o))).await; + let mut batch = Vec::with_capacity(chunk.len()); for (oid, r) in chunk.iter().zip(results) { - let o = r?; - self.write_local(oid, &o)?; + batch.push((*oid, r?)); } + self.write_local(batch).await?; } Ok(()) } - fn write_local(&self, oid: &gix_hash::oid, o: &Obj) -> Result<(), ApiError> { - if self.faulted.lock().contains(oid) { + /// Write freshly read objects into the local loose store, skipping what is + /// already faulted. Deflating an object and creating and renaming its file + /// is blocking filesystem work, so a whole batch goes to one + /// `spawn_blocking` rather than running on the tokio worker that read it + /// (principle VI: never block the async runtime). + async fn write_local(&self, batch: Vec<(ObjectId, Arc)>) -> Result<(), ApiError> { + let todo: Vec<(ObjectId, Arc)> = { + let done = self.faulted.lock(); + batch + .into_iter() + .filter(|(oid, _)| !done.contains(oid)) + .collect() + }; + if todo.is_empty() { return Ok(()); } - self.local - .write_loose_object(o.kind, oid, &o.data) - .map_err(|e| ApiError::Internal(format!("fault object {oid}: {e}")))?; - self.faulted.lock().insert(oid.to_owned()); - Ok(()) + let local = self.local.clone(); + let (written, outcome) = tokio::task::spawn_blocking(move || { + let mut written: Vec = Vec::with_capacity(todo.len()); + for (oid, o) in todo { + if let Err(e) = local.write_loose_object(o.kind, &oid, &o.data) { + let msg = format!("fault object {oid}: {e}"); + return (written, Err(ApiError::Internal(msg))); + } + written.push(oid); + } + (written, Ok(())) + }) + .await + .map_err(|e| ApiError::Internal(format!("fault write task: {e}")))?; + if !written.is_empty() { + self.faulted.lock().extend(written); + } + outcome } pub async fn kind_and_size( From 4149de91ebe36c8cc07b0ca629809d8e63c7695b Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Mon, 7 Sep 2026 12:13:40 -0400 Subject: [PATCH 27/28] Resolve the GCE machine type at startup, not in a handler gce_machine_type in crates/walgit-server/src/instance.rs forked curl inside a OnceLock::get_or_init, and info() reaches it from every readiness and health handler and the UI footer, so the first such request on an SSD host held a tokio worker for up to 300 ms on a subprocess. That is the tell principle VI names, a Command::new(...).output() on the async runtime. The probe is now a reqwest GET of the same URL with the same header, timeout and parsing, awaited once in serve() before the listener accepts, and only on the host that calls itself ssd (WALGIT_INSTANCE_KIND=ssd, or maintenance.disk = "ssd" with no explicit kind), which is the one shape whose info() prints a machine type and the only place the old code probed. Handlers now read the cell and nothing else; when the probe never ran, info() falls back to the cpu and memory shape exactly as it did when curl failed. Checked with cargo clippy -p walgit-server --all-targets --no-deps -- -D warnings, cargo test -p walgit-server --lib (the new test asserts a non-SSD host caches nothing and touches no network), cargo test -p walgit-server --test drain and cargo fmt --all -- --check. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01CSbhRh6UEjFrgBYYzucvDe --- crates/walgit-server/src/instance.rs | 115 +++++++++++++++++++++------ crates/walgit-server/src/lib.rs | 3 + 2 files changed, 93 insertions(+), 25 deletions(-) diff --git a/crates/walgit-server/src/instance.rs b/crates/walgit-server/src/instance.rs index 0eb39db..6c51b08 100644 --- a/crates/walgit-server/src/instance.rs +++ b/crates/walgit-server/src/instance.rs @@ -68,32 +68,70 @@ fn cgroup_cpus() -> Option { } None } +/// Machine type as resolved once by [`init_machine_type`]; unset until then, and +/// on every host that never probes (dev, tests, off GCP), which reads as `None`. +static MACHINE_TYPE: std::sync::OnceLock> = std::sync::OnceLock::new(); + +const MACHINE_TYPE_URL: &str = + "http://metadata.google.internal/computeMetadata/v1/instance/machine-type"; + +/// Resolve the GCE machine type once, at startup, off the request path +/// (principle VI: `info()` runs on tokio workers under every health probe). +/// +/// Only the SSD host ever shows a machine type (`info()` puts it in the shape +/// line), so only the SSD host probes, as before; every other host sends +/// nothing. The whole probe is capped at 300 ms, so an SSD host that is not on +/// GCE waits at most that, once, before it starts serving. +pub async fn init_machine_type(cfg: &walgit_config::Config) { + if !is_ssd_host(cfg) { + return; + } + let _ = MACHINE_TYPE.set(fetch_machine_type().await); +} + +/// The same reading of `WALGIT_INSTANCE_KIND` and `maintenance.disk` that +/// [`info`] uses to call a host `ssd`. +fn is_ssd_host(cfg: &walgit_config::Config) -> bool { + match std::env::var("WALGIT_INSTANCE_KIND") + .ok() + .filter(|v| !v.is_empty()) + .as_deref() + { + Some("ssd") => true, + Some("serverless" | "dev") => false, + _ => cfg.maintenance.disk == walgit_config::MaintainerDisk::Ssd, + } +} + +/// One GET at the metadata server, 300 ms for the whole thing. The value comes +/// back as a path (`projects/1234/machineTypes/c3-standard-176-lssd`); the last +/// segment is the machine type, and an empty answer is no answer. +async fn fetch_machine_type() -> Option { + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_millis(300)) + .build() + .ok()?; + let resp = client + .get(MACHINE_TYPE_URL) + .header("Metadata-Flavor", "Google") + .send() + .await + .ok()?; + if !resp.status().is_success() { + return None; + } + let body = resp.text().await.ok()?; + body.trim() + .rsplit('/') + .next() + .map(std::string::ToString::to_string) + .filter(|m| !m.is_empty()) +} + +/// The resolved machine type. A read of the cell and nothing else: handlers +/// never probe. fn gce_machine_type() -> Option { - // Cached once; 300 ms budget; only meaningful on GCE VMs (a serverless host answers - // the metadata server too but has no machine-type). - static MT: std::sync::OnceLock> = std::sync::OnceLock::new(); - MT.get_or_init(|| { - let out = std::process::Command::new("curl") - .args([ - "-sf", - "-m", - "0.3", - "-H", - "Metadata-Flavor: Google", - "http://metadata.google.internal/computeMetadata/v1/instance/machine-type", - ]) - .output() - .ok()?; - if !out.status.success() { - return None; - } - let s = String::from_utf8_lossy(&out.stdout).trim().to_string(); - s.rsplit('/') - .next() - .map(std::string::ToString::to_string) - .filter(|m| !m.is_empty()) - }) - .clone() + MACHINE_TYPE.get().cloned().flatten() } fn gib(b: u64) -> String { let g = b as f64 / (1u64 << 30) as f64; @@ -198,3 +236,30 @@ pub fn server_header(cfg: &walgit_config::Config) -> &'static str { }) .as_str() } + +#[cfg(test)] +mod tests { + use super::{gce_machine_type, init_machine_type, is_ssd_host}; + + /// A host that is not the SSD host never probes: no name to resolve, no + /// network, nothing cached, so `info()` keeps the plain cpu/memory shape. + #[tokio::test] + async fn init_machine_type_probes_only_on_the_ssd_host() { + if std::env::var("WALGIT_INSTANCE_KIND").as_deref() == Ok("ssd") { + return; // This runner calls itself the SSD host: the probe is meant to run. + } + let cfg = walgit_config::Config::default(); // maintenance.disk = tmpfs + assert!(!is_ssd_host(&cfg)); + let started = std::time::Instant::now(); + init_machine_type(&cfg).await; + let took = started.elapsed(); + assert!( + took < std::time::Duration::from_millis(100), + "the metadata server was contacted from a non-SSD host (took {took:?})" + ); + assert!( + gce_machine_type().is_none(), + "nothing should be cached when the probe never ran" + ); + } +} diff --git a/crates/walgit-server/src/lib.rs b/crates/walgit-server/src/lib.rs index 4b139e5..b0bb850 100644 --- a/crates/walgit-server/src/lib.rs +++ b/crates/walgit-server/src/lib.rs @@ -556,6 +556,9 @@ pub async fn serve( shutdown: impl Future + Send + 'static, ) -> anyhow::Result<()> { let addr = state.cfg.server.listen; + // Resolve the machine type before the first request, so `/readyz`, `/healthz` + // and the UI footer only read a cell that is already filled (principle VI). + instance::init_machine_type(&state.cfg).await; let state_for_shutdown = state.clone(); prewarm::spawn(state.clone()); bridge::spawn_sweeper(state.clone()); From 2fdedc711f2f6cf66e54206ab8ba76dfbc65997b Mon Sep 17 00:00:00 2001 From: zaeku Date: Thu, 10 Sep 2026 17:03:34 +0900 Subject: [PATCH 28/28] Spend the maintenance interval between passes, not before the first The field is documented as the pause between passes; the loop slept it at the top of the body, so a freshly started maintainer idled for one interval and a maintainer that did not live that long never ran a pass at all. The draining check stays at the top of the loop. Co-Authored-By: Claude Opus 5 --- crates/walgit-server/src/maintain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/walgit-server/src/maintain.rs b/crates/walgit-server/src/maintain.rs index 7f619bf..7d73aba 100644 --- a/crates/walgit-server/src/maintain.rs +++ b/crates/walgit-server/src/maintain.rs @@ -39,7 +39,6 @@ pub async fn run_loop(state: Arc) { let mut passes = 0u64; let mut last_unit = String::new(); loop { - tokio::time::sleep(interval).await; if walgit_wal::tasks::draining() { info!("maintenance loop: draining, no new pass"); return; @@ -97,6 +96,7 @@ pub async fn run_loop(state: Arc) { if let Err(e) = heartbeat(&state, &host, started, passes, &last_unit).await { warn!(error = %e, "maintenance heartbeat failed"); } + tokio::time::sleep(interval).await; } }