Add an opt-in open URL cache at /url/

README.md

@@ -68,6 +68,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi
 | Arch | Arch Linux | | ✗ |
 | Chef | Chef | | ✗ |
 | Generic | Any | | ✓ |
+| URL (open, immutable) | Any | | ✓ |
 | Helm | Kubernetes | | ✓ |
 | Vagrant | Vagrant | | ✗ |
 
@@ -912,6 +913,7 @@ Recently cached:
 | `GET /v2/homebrew/core/*` | Homebrew core bottle manifests and blobs from GHCR |
 | `GET /apk/{repository}/*` | Alpine APK repository protocol |
 | `GET /generic/{name}/*` | Generic HTTP download proxy (GitHub release assets, mise/aqua) |
+| `GET /url/[sha256/{hex}/]{host}/*` | Open cache of immutable https downloads, opt-in via `url_proxy.enabled` |
 | `GET /debian/*` | Debian/APT repository protocol (main archive) |
 | `GET /debian/{repository}/*` | Debian/APT repository protocol (named archive, e.g. security) |
 | `GET /rpm/*` | RPM/Yum repository protocol |

config.example.yaml

@@ -75,6 +75,22 @@ storage:
   # internal Host header or the SigV4 signature will not validate.
   # direct_serve_base_url: "https://minio.example.com"
 
+  # Base URL where the bucket serves objects anonymously, including any key
+  # prefix from the storage URL. When set, redirects point here instead of at
+  # presigned URLs, so they never expire. The bucket must allow anonymous
+  # reads under it (e.g. a bucket policy on the url/ prefix).
+  # direct_serve_public_url: "http://rgw.example.com:7480/bucket/prefix"
+
+# Open URL cache at /url/{host}/{path} (see docs/configuration.md). Caches any
+# public https URL as an immutable artifact. Off by default: it is an open
+# proxy for anyone who can reach it.
+# url_proxy:
+#   enabled: true
+#   # Redirect GETs to the stored object instead of streaming it.
+#   direct_serve: true
+#   # Bound on one whole upstream download. Default: "9m".
+#   fetch_timeout: "9m"
+
 # Database configuration
 database:
   # Database driver: "sqlite" (default) or "postgres"

docs/configuration.md

@@ -44,6 +44,7 @@ storage:
 | `storage.path` | `PROXY_STORAGE_PATH` | `-storage-path` | Local path (deprecated, use url) |
 | `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB") |
 | `storage.cache_artifacts` | `PROXY_STORAGE_CACHE_ARTIFACTS` | - | Store fetched artifacts (default: true); `false` streams them from upstream |
+| `storage.direct_serve_public_url` | `PROXY_STORAGE_DIRECT_SERVE_PUBLIC_URL` | - | Anonymous bucket URL for redirects instead of presigning (see [Open URL cache](#open-url-cache)) |
 
 `storage.max_size` counts cached artifacts only. An artifact replaced by a refetch stays in storage for at least an hour, or `storage.direct_serve_ttl` if longer, so requests already reading it can finish, and storage use can exceed the limit by what was replaced in that time.
 
@@ -58,7 +59,7 @@ storage:
 
 Every download is a fresh upstream fetch, and concurrent requests for the same artifact are not combined. Artifacts with a digest known up front (OCI blobs, Swift archives, Helm charts) are verified while streaming: the response is sent chunked, and on a mismatch the connection is aborted before the response completes so the client never receives a tampered artifact as a good one. The same happens when the upstream connection fails mid-download.
 
-`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve` or `mirror_api`, which all need stored artifacts, and the `mirror` command refuses to run with it.
+`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve`, `mirror_api` or `url_proxy.enabled`, which all need stored artifacts, and the `mirror` command refuses to run with it.
 
 ### Amazon S3
 
@@ -280,6 +281,54 @@ large mutable downloads (`releases/latest/download/...`) off this route.
 This is the cache behind [mise](https://mise.jdx.dev)'s aqua backend; see the
 mise section in the README for the client-side `url_replacements`.
 
+### Open URL cache
+
+`/generic/` only reaches configured upstreams. For build scripts that download
+pinned source tarballs from many, changing hosts, the `/url/` route caches any
+public https URL instead:
+
+```yaml
+url_proxy:
+  enabled: true        # PROXY_URL_PROXY_ENABLED
+  direct_serve: true   # PROXY_URL_PROXY_DIRECT_SERVE
+  fetch_timeout: "9m"  # PROXY_URL_PROXY_FETCH_TIMEOUT
+storage:
+  # Optional: where the bucket serves objects anonymously.
+  direct_serve_public_url: "http://rgw.example.com:7480/bucket/prefix"
+```
+
+`GET /url/{host}/{path}?{query}` fetches `https://{host}/{path}?{query}` and
+`GET /url/sha256/{hex}/{host}/{path}` also checks the download against that
+SHA-256. Every file is treated as immutable. It is fetched once, streamed into
+the artifact cache with no size limit (unlike the metadata cache), and served
+from there without revalidation, including while its host is down.
+
+- **Digest.** Pass the expected `sha256` whenever the client knows it. A download
+  that does not match is not cached and returns 502, and a cached copy with a
+  different digest is refetched. Without a digest, a URL whose content changes
+  keeps serving the first copy until it is evicted.
+- **Redirects.** With `url_proxy.direct_serve`, GET requests get a 302 to the stored
+  object, on a cache hit and right after a miss is stored, so the bytes never
+  pass through the proxy again. The target is
+  `storage.direct_serve_public_url/{storage path}` when that is set, otherwise
+  a presigned URL valid for `storage.direct_serve_ttl`. Backends that support
+  neither are streamed. HEAD is answered from the cache record and never
+  redirected. This setting is independent of `storage.direct_serve`.
+- **Expired objects.** Before redirecting, the proxy checks that the object still
+  exists. A record whose object was removed behind its back, for example by a
+  bucket lifecycle rule, is refetched once and counted in
+  `proxy_cache_missing_objects_total`.
+- **Reachability.** Only https on port 443 is fetched. The upstream client refuses
+  loopback, private and link-local addresses on every redirect hop (subject
+  to `upstream.allow_private_hosts` and `upstream.allow_loopback`), ignores
+  `HTTPS_PROXY`, and never sends `upstream.auth` credentials.
+
+The route is still an open proxy for its clients. Anyone who can reach it can
+make the proxy download and store any public file, so expose it only to
+trusted networks and set `storage.max_size`. A cold miss sends no bytes
+until the whole file is stored, so keep `fetch_timeout` below your ingress or
+load balancer's read timeout.
+
 `upstream.oci_default` sets the registry used by unprefixed `/v2` requests,
 while `upstream.oci` selects named registries through the `upstream/{name}/`
 repository prefix. For example, `oci://proxy.example.com/upstream/ghcr/owner/chart`

internal/config/config.go

@@ -150,6 +150,10 @@ type Config struct {
 	// Disabled by default to prevent unauthenticated users from triggering downloads.
 	MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"`
 
+	// URLProxy configures the open /url/ route, which caches any public
+	// https URL as an immutable artifact. Disabled by default.
+	URLProxy URLProxyConfig `json:"url_proxy" yaml:"url_proxy"`
+
 	// Gradle configures Gradle HttpBuildCache behavior.
 	Gradle GradleConfig `json:"gradle" yaml:"gradle"`
 
@@ -378,6 +382,34 @@ type StorageConfig struct {
 	// of the proxy. False is incompatible with scanning, direct_serve and
 	// mirror_api, which all depend on stored artifacts. Default: true.
 	CacheArtifacts bool `json:"cache_artifacts" yaml:"cache_artifacts"`
+	// DirectServePublicURL is the base URL under which the bucket serves
+	// stored objects anonymously, including any key prefix of the storage
+	// URL (e.g. "http://rgw:7480/bucket/prefix"). When set, redirects point
+	// at DirectServePublicURL/{storage path} instead of a presigned URL, so
+	// they never expire. The bucket must allow anonymous reads there.
+	DirectServePublicURL string `json:"direct_serve_public_url" yaml:"direct_serve_public_url"`
+}
+
+// URLProxyConfig configures the /url/ route.
+//
+// The route proxies any public https URL, so unlike upstream.generic it is an
+// open HTTP proxy for its clients: keep it on a network only trusted clients
+// reach. Fetched files are assumed immutable and cached in the artifact
+// cache, so a URL whose content changes keeps serving the first copy unless
+// the client passes the expected sha256.
+type URLProxyConfig struct {
+	// Enabled mounts the /url/ route.
+	Enabled bool `json:"enabled" yaml:"enabled"`
+
+	// DirectServe redirects GET requests to the stored object, using
+	// storage.direct_serve_public_url or, failing that, a presigned URL
+	// valid for storage.direct_serve_ttl. It applies to this route only,
+	// independently of storage.direct_serve.
+	DirectServe bool `json:"direct_serve" yaml:"direct_serve"`
+
+	// FetchTimeout bounds one whole upstream download, body included.
+	// Uses Go duration syntax. Default: "9m".
+	FetchTimeout string `json:"fetch_timeout" yaml:"fetch_timeout"`
 }
 
 // GradleConfig configures Gradle-specific features.
@@ -909,6 +941,7 @@ func (c *Config) LoadFromEnv() {
 	setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL")
 	setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL")
 	setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS")
+	setEnvString(&c.Storage.DirectServePublicURL, "PROXY_STORAGE_DIRECT_SERVE_PUBLIC_URL")
 	setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER")
 	setEnvString(&c.Database.Path, "PROXY_DATABASE_PATH")
 	setEnvString(&c.Database.URL, "PROXY_DATABASE_URL")
@@ -953,6 +986,9 @@ func (c *Config) LoadFromEnv() {
 	setEnvString(&c.Scanning.FetchBaseURL, "PROXY_SCANNING_FETCH_BASE_URL")
 	setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA")
 	setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API")
+	setEnvBool(&c.URLProxy.Enabled, "PROXY_URL_PROXY_ENABLED")
+	setEnvBool(&c.URLProxy.DirectServe, "PROXY_URL_PROXY_DIRECT_SERVE")
+	setEnvString(&c.URLProxy.FetchTimeout, "PROXY_URL_PROXY_FETCH_TIMEOUT")
 	setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL")
 	setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE")
 	setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT")
@@ -974,6 +1010,33 @@ func validateAbsoluteURL(fieldName, value string) error {
 	return nil
 }
 
+// validateHTTPURL is validateAbsoluteURL restricted to http and https, with no
+// query or fragment, for URLs that object paths are appended to.
+func validateHTTPURL(fieldName, value string) error {
+	u, err := url.Parse(value)
+	if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" ||
+		u.RawQuery != "" || u.Fragment != "" {
+		return fmt.Errorf("invalid %s %q: must be an http or https URL without query or fragment", fieldName, value)
+	}
+	return nil
+}
+
+// validateURLProxy checks the /url/ route settings and the public object URL
+// its redirects use.
+func (c *Config) validateURLProxy() error {
+	if c.Storage.DirectServePublicURL != "" {
+		if err := validateHTTPURL("storage.direct_serve_public_url", c.Storage.DirectServePublicURL); err != nil {
+			return err
+		}
+	}
+	if c.URLProxy.FetchTimeout != "" {
+		if d, err := time.ParseDuration(c.URLProxy.FetchTimeout); err != nil || d <= 0 {
+			return fmt.Errorf("invalid url_proxy.fetch_timeout %q: must be a positive duration", c.URLProxy.FetchTimeout)
+		}
+	}
+	return nil
+}
+
 // Validate checks the configuration for errors.
 func (c *Config) Validate() error {
 	if c.Listen == "" {
@@ -1077,6 +1140,8 @@ func (c *Config) validateCacheArtifacts() error {
 		return fmt.Errorf("storage.cache_artifacts: false cannot be combined with storage.direct_serve: no artifacts are stored to redirect to")
 	case c.MirrorAPI:
 		return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror_api: mirrored artifacts would never be served")
+	case c.URLProxy.Enabled:
+		return fmt.Errorf("storage.cache_artifacts: false cannot be combined with url_proxy.enabled: /url/ serves only stored artifacts")
 	}
 	return nil
 }
@@ -1086,6 +1151,10 @@ func (c *Config) validateComponents() error {
 		return err
 	}
 
+	if err := c.validateURLProxy(); err != nil {
+		return err
+	}
+
 	if err := c.Upstream.Validate(); err != nil {
 		return err
 	}
@@ -1337,6 +1406,23 @@ func (c *Config) ParseGradleBuildCacheSweepInterval() time.Duration {
 	return d
 }
 
+// defaultURLProxyFetchTimeout stays under the 10 minute read timeout common
+// on ingress controllers, so a slow fetch fails here with a clear error.
+const defaultURLProxyFetchTimeout = 9 * time.Minute
+
+// ParseURLProxyFetchTimeout returns the /url/ upstream fetch timeout.
+// Returns 9 minutes if unset or invalid.
+func (c *Config) ParseURLProxyFetchTimeout() time.Duration {
+	if c.URLProxy.FetchTimeout == "" {
+		return defaultURLProxyFetchTimeout
+	}
+	d, err := time.ParseDuration(c.URLProxy.FetchTimeout)
+	if err != nil || d <= 0 {
+		return defaultURLProxyFetchTimeout
+	}
+	return d
+}
+
 // ParseDirectServeTTL returns the presigned URL expiry duration.
 // Returns 15 minutes if unset.
 func (c *Config) ParseDirectServeTTL() time.Duration {

internal/config/config_test.go

@@ -1369,6 +1369,7 @@ func TestValidateCacheArtifactsDisabled(t *testing.T) {
 		{"with scanning", func(c *Config) { c.Scanning.Enabled = true }, "scanning.enabled"},
 		{"with direct_serve", func(c *Config) { c.Storage.DirectServe = true }, "storage.direct_serve"},
 		{"with mirror_api", func(c *Config) { c.MirrorAPI = true }, "mirror_api"},
+		{"with url_proxy", func(c *Config) { c.URLProxy.Enabled = true }, "url_proxy.enabled"},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -1416,3 +1417,51 @@ func TestLoadCacheArtifactsFromEnv(t *testing.T) {
 		t.Error("Storage.CacheArtifacts should be false")
 	}
 }
+
+func TestValidateDirectServePublicURL(t *testing.T) {
+	for _, bad := range []string{"bucket/prefix", "s3://bucket", "http://rgw/bucket?x=1", "http://rgw/bucket#f"} {
+		cfg := Default()
+		cfg.Storage.DirectServePublicURL = bad
+		if err := cfg.Validate(); err == nil {
+			t.Errorf("direct_serve_public_url %q: expected validation error", bad)
+		}
+	}
+	cfg := Default()
+	cfg.Storage.DirectServePublicURL = "http://bucket.internal:7480/goproxy/pkgproxy"
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("valid direct_serve_public_url: %v", err)
+	}
+}
+
+func TestURLProxyFetchTimeout(t *testing.T) {
+	cfg := Default()
+	if got := cfg.ParseURLProxyFetchTimeout(); got != defaultURLProxyFetchTimeout {
+		t.Errorf("default = %v", got)
+	}
+	cfg.URLProxy.FetchTimeout = "20m"
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("valid fetch_timeout: %v", err)
+	}
+	if got := cfg.ParseURLProxyFetchTimeout(); got != 20*time.Minute {
+		t.Errorf("parsed = %v", got)
+	}
+	for _, bad := range []string{"soon", "0", "-1m"} {
+		cfg.URLProxy.FetchTimeout = bad
+		if err := cfg.Validate(); err == nil {
+			t.Errorf("fetch_timeout %q: expected validation error", bad)
+		}
+	}
+}
+
+func TestLoadFromEnvURLProxy(t *testing.T) {
+	t.Setenv("PROXY_URL_PROXY_ENABLED", "true")
+	t.Setenv("PROXY_URL_PROXY_DIRECT_SERVE", "true")
+	t.Setenv("PROXY_URL_PROXY_FETCH_TIMEOUT", "3m")
+	t.Setenv("PROXY_STORAGE_DIRECT_SERVE_PUBLIC_URL", "http://rgw:7480/b/p")
+	cfg := Default()
+	cfg.LoadFromEnv()
+	if !cfg.URLProxy.Enabled || !cfg.URLProxy.DirectServe || cfg.URLProxy.FetchTimeout != "3m" ||
+		cfg.Storage.DirectServePublicURL != "http://rgw:7480/b/p" {
+		t.Errorf("got url_proxy %+v, public url %q", cfg.URLProxy, cfg.Storage.DirectServePublicURL)
+	}
+}

internal/handler/handler.go

@@ -95,6 +95,8 @@ func packagePURLStrings(ecosystem, name, version string) (string, string, error)
 
 const contentTypeJSON = "application/json"
 
+const contentTypeOctetStream = "application/octet-stream"
+
 const (
 	headerAccept          = "Accept"
 	headerAcceptEncoding  = "Accept-Encoding"
@@ -172,8 +174,11 @@ type Proxy struct {
 	// URLs so clients receive a public address even when the proxy reaches
 	// storage at an internal one.
 	DirectServeBaseURL string
-	HTTPClient         *http.Client
-	AuthForURL         func(string) (headerName, headerValue string)
+	// DirectServePublicURL, if set, is where the bucket serves objects
+	// anonymously; redirects point under it instead of at presigned URLs.
+	DirectServePublicURL string
+	HTTPClient           *http.Client
+	AuthForURL           func(string) (headerName, headerValue string)
 
 	// StreamArtifacts streams artifacts from upstream without storing them.
 	// Each request fetches its own copy: there is no cache to check and
@@ -334,9 +339,9 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s
 	}
 
 	if p.DirectServe {
-		signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL)
+		redirect, err := p.directServeURL(ctx, artifact.StoragePath)
 		if err == nil {
-			result.RedirectURL = rewriteSignedURLHost(signed, p.DirectServeBaseURL)
+			result.RedirectURL = redirect
 			p.recordCacheHit(artifact.Ecosystem, versionPURL, filename)
 			return result, nil
 		}
@@ -375,6 +380,21 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s
 	return result, nil
 }
 
+// directServeURL returns an address clients can download storagePath from
+// directly: under DirectServePublicURL when the bucket serves it anonymously,
+// otherwise a presigned URL. It returns storage.ErrSignedURLUnsupported when
+// the backend can do neither.
+func (p *Proxy) directServeURL(ctx context.Context, storagePath string) (string, error) {
+	if p.DirectServePublicURL != "" {
+		return storage.PublicObjectURL(p.DirectServePublicURL, storagePath), nil
+	}
+	signed, err := p.Storage.SignedURL(ctx, storagePath, p.DirectServeTTL)
+	if err != nil {
+		return "", err
+	}
+	return rewriteSignedURLHost(signed, p.DirectServeBaseURL), nil
+}
+
 // rewriteSignedURLHost replaces the scheme and host of a signed URL with those
 // from baseURL, preserving the path and query (which carry the signature).
 // Returns signed unchanged if baseURL is empty or either URL fails to parse.
@@ -534,9 +554,10 @@ func (p *Proxy) openStoredArtifact(ctx context.Context, artifact artifacts.Artif
 	}
 
 	return &CacheResult{
-		Reader:   reader,
-		Artifact: artifact,
-		Cached:   false,
+		Reader:      reader,
+		Artifact:    artifact,
+		Cached:      false,
+		storagePath: storagePath,
 	}, nil
 }