From aa2741561a130d5b5fea9016364a53ba6cb2c704 Mon Sep 17 00:00:00 2001 From: pageman <24869+pageman@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:47:01 +0000 Subject: [PATCH] fix: harden storage coordination and browser auth --- crates/walgit-config/src/lib.rs | 33 ++------- crates/walgit-git/src/lib.rs | 12 +++- crates/walgit-server/build.rs | 14 ++-- crates/walgit-server/src/health.rs | 11 +-- crates/walgit-server/src/lfs.rs | 11 ++- crates/walgit-server/src/prewarm.rs | 35 ++++++--- crates/walgit-server/src/web/v1.rs | 42 +++++++++-- crates/walgit-server/tests/sim.rs | 14 +++- crates/walgit-store/src/gcs.rs | 36 +++++++--- crates/walgit-store/src/s3.rs | 107 +++++++++++++--------------- web/API.md | 4 +- web/sdk/README.md | 2 +- web/sdk/repos.ts | 2 +- 13 files changed, 194 insertions(+), 129 deletions(-) diff --git a/crates/walgit-config/src/lib.rs b/crates/walgit-config/src/lib.rs index 9132dc0..672356b 100644 --- a/crates/walgit-config/src/lib.rs +++ b/crates/walgit-config/src/lib.rs @@ -9,7 +9,7 @@ pub use bytesize::ByteSize; use serde::{Deserialize, Serialize}; pub use std::str::FromStr; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] pub struct Config { pub server: ServerConfig, @@ -469,7 +469,7 @@ impl Default for MaintenanceConfig { /// everywhere, so the edge's read-only fallback (D29) works. /// * **maintain**: the maintainer loop's units (checkpoints, bundles, compaction, /// fsck/repair) — only on hosts with the `maintain` role. -/// Placement is by rule, not by capacity: a repo is either this host's or not. +/// Placement is by rule, not by capacity: a repo is either this host's or not. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] pub struct PlacementConfig { @@ -857,7 +857,7 @@ impl Config { /// never `upstream.token_env` (that name is host-only). pub fn public_settings_toml(&self) -> Result { let mut doc: toml::Table = toml::Table::try_from(self).context("serializing config")?; - doc.retain(|k, _| SETTINGS_SECTIONS.iter().any(|s| *s == k)); + doc.retain(|k, _| SETTINGS_SECTIONS.contains(&k)); if let Some(toml::Value::Table(u)) = doc.get_mut("upstream") { u.remove("token_env"); } @@ -1009,25 +1009,6 @@ pub fn repo_listed(list: &[String], owner: &str, name: &str) -> bool { }) } -impl Default for Config { - fn default() -> Self { - Config { - server: ServerConfig::default(), - store: StoreConfig::default(), - cache: CacheConfig::default(), - wal: WalConfig::default(), - compaction: CompactionConfig::default(), - maintenance: MaintenanceConfig::default(), - bundles: BundlesConfig::default(), - placement: PlacementConfig::default(), - lfs: LfsConfig::default(), - upstream: UpstreamConfig::default(), - git: GitConfig::default(), - telemetry: TelemetryConfig::default(), - events: EventsConfig::default(), - } - } -} impl Default for ServerConfig { fn default() -> Self { ServerConfig { @@ -1371,10 +1352,10 @@ impl Config { self.server.listen.set_port(port); // Standalone / `dev server`: public_url is the origin the browser hits. Keep its // port in lockstep with PORT. A real public_url is left alone. - if let Some(u) = self.server.public_url.as_mut() { - if origin_is_loopback(u) { - *u = rewrite_origin_port(u, port); - } + if let Some(u) = self.server.public_url.as_mut() + && origin_is_loopback(u) + { + *u = rewrite_origin_port(u, port); } } Ok(ignored) diff --git a/crates/walgit-git/src/lib.rs b/crates/walgit-git/src/lib.rs index d48fa36..016fe7b 100644 --- a/crates/walgit-git/src/lib.rs +++ b/crates/walgit-git/src/lib.rs @@ -2873,7 +2873,7 @@ fn capabilities_for(service: Service, format: ObjectFormat) -> String { let agent = format!("agent=walgit/{WALGIT_VERSION}"); match service { Service::UploadPack => format!( - "multi_ack_detailed side-band-64k thin-pack ofs-delta shallow deepen-since deepen-not \ + "multi_ack_detailed side-band-64k thin-pack ofs-delta shallow \ no-progress include-tag allow-tip-sha1-in-want allow-reachable-sha1-in-want filter \ object-format={of} {agent}" ), @@ -3609,6 +3609,16 @@ mod index_pack_trace_tests { ); } + #[test] + fn upload_pack_does_not_advertise_unimplemented_deepen_modes() { + let caps = capabilities_for(Service::UploadPack, ObjectFormat::Sha1); + assert!(!caps.split_whitespace().any(|c| c == "deepen-since")); + assert!(!caps.split_whitespace().any(|c| c == "deepen-not")); + assert!(!caps.split_whitespace().any(|c| c == "packfile-uris")); + assert!(caps.split_whitespace().any(|c| c == "shallow")); + assert!(caps.split_whitespace().any(|c| c == "filter")); + } + #[test] fn successful_index_pack_records_non_zero_git_ms() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/walgit-server/build.rs b/crates/walgit-server/build.rs index f7a7eb4..f34bb34 100644 --- a/crates/walgit-server/build.rs +++ b/crates/walgit-server/build.rs @@ -1,8 +1,7 @@ //! Make `cargo build`/`cargo test` work in a fresh checkout without running the -//! web build. `rust-embed` requires `../../web/dist` to exist at compile time; -//! when the SPA has not been built (`just web-build`) we drop in a placeholder -//! `index.html` so the server compiles and the HTML routes still answer 200. -//! A real `pnpm run build` overwrites the placeholder. +//! web build. `rust-embed` requires `../../web/dist` to exist at compile time. +//! Development builds may use a placeholder when the SPA has not been built; +//! release builds fail instead so a deployable artifact cannot silently omit UI. use std::fs; use std::path::Path; @@ -15,12 +14,17 @@ fn main() { let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); let dist = manifest.join("../../web/dist"); println!("cargo:rerun-if-changed={}", dist.display()); + println!("cargo:rerun-if-env-changed=PROFILE"); let index = dist.join("index.html"); if !index.exists() { + let release = std::env::var("PROFILE").as_deref() == Ok("release"); + if release { + panic!("web/dist/index.html is missing in a release build; run `just web-build` first"); + } fs::create_dir_all(&dist).expect("create web/dist"); fs::write(&index, PLACEHOLDER).expect("write placeholder web/dist/index.html"); println!( - "cargo:warning=web/dist was missing; wrote a placeholder index.html (run `just web-build` for the real UI)" + "cargo:warning=web/dist was missing; wrote a development placeholder index.html (run `just web-build` for the real UI)" ); } } diff --git a/crates/walgit-server/src/health.rs b/crates/walgit-server/src/health.rs index 96c22c3..d32ddf3 100644 --- a/crates/walgit-server/src/health.rs +++ b/crates/walgit-server/src/health.rs @@ -18,28 +18,29 @@ pub async fn healthz() -> Json { Json(json!({"status": "ok", "version": BUILD_SHA})) } -/// 200 once startup prewarm (`cache.prewarm`) finished or -/// `cache.prewarm_ready_timeout` elapsed; 503 (with what is pending) before. +/// 200 once every required startup prewarm (`cache.prewarm`) succeeds; +/// 503 while warming or when any required prewarm failed. pub async fn readyz(State(state): State>) -> Response { let r = &state.readiness; let pending = r.pending.load(std::sync::atomic::Ordering::Acquire); + let failures = r.failed.load(std::sync::atomic::Ordering::Acquire); // Draining after SIGTERM: tell the edge/LB to stop routing here at once // (in-flight work finishes; new object work is refused with Retry-After). if walgit_wal::tasks::shutting_down() { return (StatusCode::SERVICE_UNAVAILABLE, [(axum::http::header::RETRY_AFTER, "15")], Json(json!({"status": "draining", "version": BUILD_SHA, "running": state.registry.tasks().running_all().len(), "instance": crate::instance::info(&state.cfg)}))).into_response(); } - if r.ready(state.cfg.cache.prewarm_ready_timeout) { + if r.ready() { // Placement is a liveness fact: deployment verification should assert // that each important repository is served by at least one ready host. // Return rules, not repository lists; /readyz remains open for probes. let p = &state.cfg.placement; - return Json(json!({"status": "ready", "version": BUILD_SHA, "prewarm_pending": pending, "instance": crate::instance::info(&state.cfg), + return Json(json!({"status": "ready", "version": BUILD_SHA, "prewarm_pending": pending, "prewarm_failed": failures, "instance": crate::instance::info(&state.cfg), "placement": {"serve": p.serve, "serve_exclude": p.serve_exclude, "maintain": p.maintain, "maintain_exclude": p.maintain_exclude}})).into_response(); } ( StatusCode::SERVICE_UNAVAILABLE, // Unauthenticated (startup probe): counts only, no repo names. - Json(json!({"status": "warming", "version": BUILD_SHA, "prewarm_pending": pending, "running": state.registry.tasks().running_all().len(), "instance": crate::instance::info(&state.cfg)})), + Json(json!({"status": if failures > 0 { "prewarm_failed" } else { "warming" }, "version": BUILD_SHA, "prewarm_pending": pending, "prewarm_failed": failures, "running": state.registry.tasks().running_all().len(), "instance": crate::instance::info(&state.cfg)})), ) .into_response() } diff --git a/crates/walgit-server/src/lfs.rs b/crates/walgit-server/src/lfs.rs index d6b3f29..43e1d4c 100644 --- a/crates/walgit-server/src/lfs.rs +++ b/crates/walgit-server/src/lfs.rs @@ -82,7 +82,16 @@ pub async fn batch( if !st.cfg.lfs.enabled { return Err(ApiError::NotFound("lfs disabled".into())); } - let _ = st.auth.require_read(headers).await.map_err(auth_err)?; + if body.operation == "upload" { + let _ = st.auth.require_write(headers).await.map_err(auth_err)?; + } else if body.operation == "download" { + let _ = st.auth.require_read(headers).await.map_err(auth_err)?; + } else { + return Err(ApiError::BadRequest(format!( + "unsupported lfs operation: {}", + body.operation + ))); + } not_served_here(st, &route.id)?; let handle = open_repo(st, &route.id, false).await?; let store = handle.store().clone(); diff --git a/crates/walgit-server/src/prewarm.rs b/crates/walgit-server/src/prewarm.rs index cf14bf5..faa7191 100644 --- a/crates/walgit-server/src/prewarm.rs +++ b/crates/walgit-server/src/prewarm.rs @@ -2,8 +2,8 @@ //! to this instance (packs when they fit, the remote pack indexes otherwise) //! and touch the default branch's root tree, so the first user request on a //! fresh instance finds everything in place. Each repo is a `prewarm` task -//! (discoverable at `…/tasks`); `/readyz` can be gated on completion -//! (`cache.prewarm_ready_timeout`). +//! (discoverable at `…/tasks`); `/readyz` is ready only after every required +//! prewarm succeeds. use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -13,9 +13,11 @@ use tracing::Instrument; use crate::AppState; pub struct Readiness { - /// All prewarms finished (ok or not). + /// All configured prewarms finished successfully. pub done: AtomicBool, pub pending: AtomicUsize, + /// Number of prewarm tasks that finished with an error in the current run. + pub failed: AtomicUsize, pub started_at: Instant, } @@ -24,14 +26,14 @@ impl Readiness { Arc::new(Readiness { done: AtomicBool::new(true), pending: AtomicUsize::new(0), + failed: AtomicUsize::new(0), started_at: Instant::now(), }) } - /// True when traffic may be routed here. - pub fn ready(&self, timeout: std::time::Duration) -> bool { - self.done.load(Ordering::Acquire) - || timeout.is_zero() - || self.started_at.elapsed() >= timeout + /// True only when every configured prewarm completed successfully. + /// + pub fn ready(&self) -> bool { + self.done.load(Ordering::Acquire) && self.failed.load(Ordering::Acquire) == 0 } } @@ -40,6 +42,7 @@ impl Default for Readiness { Readiness { done: AtomicBool::new(true), pending: AtomicUsize::new(0), + failed: AtomicUsize::new(0), started_at: Instant::now(), } } @@ -52,6 +55,7 @@ pub fn spawn(state: Arc) { return; } state.readiness.done.store(false, Ordering::Release); + state.readiness.failed.store(0, Ordering::Release); state .readiness .pending @@ -68,7 +72,10 @@ pub fn spawn(state: Arc) { let t = Instant::now(); match warm(&st, &r).await { Ok(summary) => tracing::info!(repo = %r, elapsed_ms = t.elapsed().as_millis() as u64, "prewarm: {summary}"), - Err(e) => tracing::warn!(repo = %r, elapsed_ms = t.elapsed().as_millis() as u64, "prewarm failed: {e}"), + Err(e) => { + st.readiness.failed.fetch_add(1, Ordering::AcqRel); + tracing::warn!(repo = %r, elapsed_ms = t.elapsed().as_millis() as u64, "prewarm failed: {e}"); + } } st.readiness.pending.fetch_sub(1, Ordering::AcqRel); })); @@ -77,7 +84,15 @@ pub fn spawn(state: Arc) { let _ = h.await; } state.readiness.done.store(true, Ordering::Release); - tracing::info!("prewarm complete; instance ready"); + let failures = state.readiness.failed.load(Ordering::Acquire); + if failures == 0 { + tracing::info!("prewarm complete; instance ready"); + } else { + tracing::error!( + failures, + "prewarm complete with failures; instance remains unready" + ); + } }); } diff --git a/crates/walgit-server/src/web/v1.rs b/crates/walgit-server/src/web/v1.rs index 49813f8..ec4f9b3 100644 --- a/crates/walgit-server/src/web/v1.rs +++ b/crates/walgit-server/src/web/v1.rs @@ -16,13 +16,13 @@ use std::sync::Arc; use axum::{ Router, body::Body, - extract::{Path, Request, State}, + extract::{Path, Query, Request, State}, http::{HeaderMap, HeaderValue, Method, StatusCode, header}, middleware::Next, response::{Html, IntoResponse, Response}, routing::{get, post}, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::repo::RepoRoute; use crate::web::api::{Need, RefInfo, etag_for, json_swr, run}; @@ -274,14 +274,31 @@ async fn me(State(st): State>, headers: HeaderMap) -> Response { } } +#[derive(Deserialize, Default)] +struct AuthenticateQuery { + /// Origin of the SDK page that opened this popup. It is accepted only when + /// it is already allowed by the server's credentialed CORS policy. + origin: Option, +} + /// `GET /api/v1/authenticate`: the popup landing page of the browser lane. /// `require_auth` sends an unauthenticated browser through sign-in first; this /// authenticated page then tells its opener and closes. The SDK opens it when a /// browser-lane call answers 401. -async fn authenticate(State(st): State>, headers: HeaderMap) -> Response { +async fn authenticate( + State(st): State>, + headers: HeaderMap, + Query(query): Query, +) -> Response { match st.auth.require_read(&headers).await { Ok(p) => { - let page = AUTHENTICATE_HTML.replace("{{principal}}", &html_escape(&p.name)); + let target_origin = query + .origin + .filter(|origin| origin_allowed(&st.cfg, origin)) + .unwrap_or_else(|| crate::smart::request_base_url(&st, &headers)); + let page = AUTHENTICATE_HTML + .replace("{{principal}}", &html_escape(&p.name)) + .replace("{{target_origin}}", &html_escape(&target_origin)); let mut r = Html(page).into_response(); r.headers_mut() .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); @@ -300,8 +317,8 @@ const AUTHENTICATE_HTML: &str = r#"