From ef2636695d407bd4c68a3bd9c10a014e3282e9e6 Mon Sep 17 00:00:00 2001 From: "Aaron K. Clark (CryptoJones)" Date: Sat, 5 Sep 2026 19:47:41 -0500 Subject: [PATCH 1/4] Add nvpair-shared/ingressauth, an opt-in API-key gate for non-loopback plaintext callers Both inference proxies refuse plaintext requests that do not arrive from loopback. This package is the credential gate they will share so a LAN caller can be admitted only when the operator opts in by configuring API keys (NVPAIR_PROXY_API_KEYS_FILE, default /proxy-api-keys, or NVPAIR_PROXY_API_KEYS) and the caller presents one as Authorization: Bearer or X-Api-Key, optionally restricted by NVPAIR_PROXY_ALLOWED_CIDRS. Keys are held only as SHA-256 digests and compared in constant time across every configured digest with no early exit. Every failure fails closed: a key file readable by other users, a malformed entry, an unreadable file, or a malformed CIDR contributes no keys. The key file is re-read when its size, modification time, or mode changes, so keys can be rotated or revoked without a restart. A rejected key is logged only as an eight-hex-digit digest fingerprint. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01BwxtwuoRxP75PdR6NAmMS3 Signed-off-by: Aaron K. Clark (CryptoJones) --- services/shared/ingressauth/ingressauth.go | 461 ++++++++++++++++++ .../shared/ingressauth/ingressauth_test.go | 456 +++++++++++++++++ 2 files changed, 917 insertions(+) create mode 100644 services/shared/ingressauth/ingressauth.go create mode 100644 services/shared/ingressauth/ingressauth_test.go diff --git a/services/shared/ingressauth/ingressauth.go b/services/shared/ingressauth/ingressauth.go new file mode 100644 index 00000000..f6ad20a6 --- /dev/null +++ b/services/shared/ingressauth/ingressauth.go @@ -0,0 +1,461 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package ingressauth is the opt-in credential gate the inference proxies apply +// to a plaintext request that did not arrive from loopback. Both proxies share +// it for the same reason they share nvpair-shared/cors: the two must accept and +// refuse an outside caller identically, and one implementation is what keeps +// them from drifting. +// +// With nothing configured the gate is disabled and the proxies keep their +// loopback-only behavior — a LAN caller is refused before this package is +// consulted. An operator enables it by configuring at least one API key, either +// in a key file (NVPAIR_PROXY_API_KEYS_FILE, default /proxy-api-keys) or +// inline (NVPAIR_PROXY_API_KEYS). Once enabled, a non-loopback plaintext caller +// must present a configured key as "Authorization: Bearer " or, for clients +// built on the Anthropic SDK convention, "X-Api-Key: "; an optional +// NVPAIR_PROXY_ALLOWED_CIDRS narrows which source addresses may even try. +// Loopback callers are never asked for a key — the desktop application, the +// terminal interface, and local tools are unaffected by enabling the gate. +// +// Every failure fails closed. A key file that cannot be read, is readable by +// other users, or contains an entry that could never match over the wire +// contributes no keys, the reason is logged, and the LAN stays closed. Keys are +// held in memory only as SHA-256 digests and are compared in constant time; a +// rejected credential is logged as a short digest fingerprint, never as itself. +package ingressauth + +import ( + "bufio" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "net/http" + "net/netip" + "os" + "runtime" + "strings" + "sync" + "time" + "unicode" + + "nvpair-shared/appdir" +) + +const ( + // EnvKeys holds comma-separated API keys, for headless and container + // deployments where a file is inconvenient. Combined with the key file. + EnvKeys = "NVPAIR_PROXY_API_KEYS" + // EnvKeysFile overrides the key file path. Unset, the file is + // /DefaultKeyFileName and is consulted only if it exists. + EnvKeysFile = "NVPAIR_PROXY_API_KEYS_FILE" + // EnvAllowedCIDRs optionally lists comma-separated CIDR prefixes an + // authenticated non-loopback caller must originate from. + EnvAllowedCIDRs = "NVPAIR_PROXY_ALLOWED_CIDRS" + // DefaultKeyFileName is the key file's name inside the application data + // directory (see nvpair-shared/appdir). + DefaultKeyFileName = "proxy-api-keys" + + // MinKeyLength is the shortest key the gate accepts. A 32-character key + // drawn from hex already carries 128 bits, which puts online guessing out + // of reach without a lockout mechanism. + MinKeyLength = 32 + + // CodeUnauthorized is the ingress error code for a missing or wrong key. + CodeUnauthorized = "unauthorized" + // CodeSourceNotAllowed is the ingress error code for a caller outside the + // configured CIDR allowlist. + CodeSourceNotAllowed = "source-not-allowed" + + headerAuthorization = "Authorization" + headerAPIKey = "X-Api-Key" + bearerScheme = "bearer" + noCredential = "none" + + unauthorizedMessage = "a valid API key is required for non-loopback requests; " + + "send it as Authorization: Bearer or X-Api-Key: " +) + +// Decision is the gate's verdict on one request. When Allowed is false, Status, +// Code, and Message are what the proxy should answer with, in the same shape as +// its other ingress rejections. KeyFingerprint identifies the presented key for +// the log without revealing it; it is "none" when no credential was sent. +type Decision struct { + Allowed bool + Status int + Code string + Message string + KeyFingerprint string +} + +type digest = [sha256.Size]byte + +// fileStamp is the part of a key file's metadata that decides whether it must +// be re-read. Mode is included because fixing permissions with chmod changes +// neither size nor modification time, yet must take effect. +type fileStamp struct { + size int64 + modTime time.Time + mode fs.FileMode + exists bool +} + +// Gate holds the configured credentials and allowlist. Its zero value is a +// disabled gate; construct one with FromEnv or New. +type Gate struct { + mu sync.Mutex + + // inline keys come from the environment (or New) and never change. + inline []digest + // broken records an unrecoverable configuration error in the environment + // (a malformed inline key or CIDR). The gate then stays disabled for the + // life of the process, regardless of the key file. + broken bool + + // filePath, when non-empty, is re-checked on every Enabled call so keys can + // be rotated without restarting the proxy. explicitFile records that the + // operator named the path, so its absence is worth reporting. + filePath string + explicitFile bool + fileKeys []digest + fileStamp fileStamp + fileChecked bool + + cidrs []netip.Prefix + + // announced* remember the last state written to the log, so a change is + // reported once rather than on every request. + announcedOnce bool + announcedEnabled bool + announcedKeys int +} + +// New builds a gate from literal keys and prefixes, for tests and callers that +// resolve configuration themselves. Keys are validated like file entries; an +// invalid key panics, since a caller passing literals has a programming error +// rather than an operator mistake. +func New(keys []string, cidrs []netip.Prefix) *Gate { + g := &Gate{cidrs: cidrs} + for _, k := range keys { + if err := validateKey(k); err != nil { + panic("ingressauth.New: " + err.Error()) + } + g.inline = append(g.inline, sha256.Sum256([]byte(k))) + } + g.mu.Lock() + g.announceLocked() + g.mu.Unlock() + return g +} + +// FromEnv builds the gate from the process environment. It never fails: a +// configuration error is logged and yields a gate that stays disabled, which +// leaves the proxy in its loopback-only default. +func FromEnv() *Gate { + g := &Gate{} + + if raw := os.Getenv(EnvKeys); strings.TrimSpace(raw) != "" { + for i, k := range strings.Split(raw, ",") { + k = strings.TrimSpace(k) + if k == "" { + continue + } + if err := validateKey(k); err != nil { + slog.Error("authenticated LAN ingress disabled: invalid inline API key", + "env", EnvKeys, "entry", i+1, "err", err) + g.broken = true + break + } + g.inline = append(g.inline, sha256.Sum256([]byte(k))) + } + } + + if raw := os.Getenv(EnvAllowedCIDRs); strings.TrimSpace(raw) != "" { + for _, s := range strings.Split(raw, ",") { + s = strings.TrimSpace(s) + if s == "" { + continue + } + prefix, err := netip.ParsePrefix(s) + if err != nil { + slog.Error("authenticated LAN ingress disabled: invalid CIDR allowlist entry", + "env", EnvAllowedCIDRs, "entry", s, "err", err) + g.broken = true + break + } + g.cidrs = append(g.cidrs, prefix.Masked()) + } + } + + if p := os.Getenv(EnvKeysFile); p != "" { + g.filePath = p + g.explicitFile = true + } else if p, err := appdir.Path(DefaultKeyFileName); err == nil { + g.filePath = p + } else { + slog.Warn("authenticated LAN ingress: cannot resolve the default key file location", "err", err) + } + + g.mu.Lock() + g.refreshLocked() + g.announceLocked() + g.mu.Unlock() + return g +} + +// Enabled reports whether at least one API key is configured, re-reading the +// key file first if it changed. The proxy consults this per non-loopback +// request, so adding, rotating, or removing keys needs no restart. +func (g *Gate) Enabled() bool { + g.mu.Lock() + defer g.mu.Unlock() + g.refreshLocked() + g.announceLocked() + return g.enabledLocked() +} + +func (g *Gate) enabledLocked() bool { + return !g.broken && len(g.inline)+len(g.fileKeys) > 0 +} + +// Authorize decides whether a non-loopback plaintext request may proceed. The +// allowlist is checked before the credential, so a caller outside it learns +// nothing about whether its key is valid. Authorize does not write to the +// response; the proxy does, in its own error format. +func (g *Gate) Authorize(r *http.Request) Decision { + g.mu.Lock() + g.refreshLocked() + cidrs := g.cidrs + digests := make([]digest, 0, len(g.inline)+len(g.fileKeys)) + digests = append(digests, g.inline...) + digests = append(digests, g.fileKeys...) + enabled := g.enabledLocked() + g.mu.Unlock() + + if !enabled { + // The proxy only asks an enabled gate; answer conservatively anyway. + return Decision{Status: http.StatusForbidden, Code: CodeSourceNotAllowed, + Message: "authenticated LAN ingress is not enabled", KeyFingerprint: noCredential} + } + + if len(cidrs) > 0 { + ip, ok := remoteAddr(r) + if !ok || !anyPrefixContains(cidrs, ip) { + return Decision{Status: http.StatusForbidden, Code: CodeSourceNotAllowed, + Message: "the caller's address is outside " + EnvAllowedCIDRs, KeyFingerprint: noCredential} + } + } + + cred, ok := credentialFrom(r) + if !ok { + return Decision{Status: http.StatusUnauthorized, Code: CodeUnauthorized, + Message: unauthorizedMessage, KeyFingerprint: noCredential} + } + if !matchesAny(digests, sha256.Sum256([]byte(cred))) { + return Decision{Status: http.StatusUnauthorized, Code: CodeUnauthorized, + Message: unauthorizedMessage, KeyFingerprint: Fingerprint(cred)} + } + return Decision{Allowed: true, Status: http.StatusOK, KeyFingerprint: Fingerprint(cred)} +} + +// StripCredential removes the presented key from a request the gate admitted, +// so the proxy's credential is never forwarded to an engine or a peer. +func (g *Gate) StripCredential(h http.Header) { + h.Del(headerAuthorization) + h.Del(headerAPIKey) +} + +// Fingerprint returns the first eight hex characters of a key's SHA-256 digest: +// enough for an operator to tell repeated rejections of one misconfigured +// client apart from a scan, without the log ever holding the key. +func Fingerprint(key string) string { + sum := sha256.Sum256([]byte(key)) + return hex.EncodeToString(sum[:4]) +} + +// matchesAny compares the presented digest against every configured digest in +// constant time and without an early exit, so neither the key length nor the +// position of a match is observable through timing. +func matchesAny(configured []digest, presented digest) bool { + match := 0 + for i := range configured { + match |= subtle.ConstantTimeCompare(configured[i][:], presented[:]) + } + return match == 1 +} + +// credentialFrom extracts the client's key: a Bearer token first, then the +// X-Api-Key header. A query parameter is deliberately not accepted, because +// URLs end up in access logs and browser histories. +func credentialFrom(r *http.Request) (string, bool) { + if auth := strings.TrimSpace(r.Header.Get(headerAuthorization)); auth != "" { + scheme, token, found := strings.Cut(auth, " ") + if found && strings.EqualFold(scheme, bearerScheme) { + if token = strings.TrimSpace(token); token != "" { + return token, true + } + } + } + if key := strings.TrimSpace(r.Header.Get(headerAPIKey)); key != "" { + return key, true + } + return "", false +} + +// remoteAddr parses the transport-level peer address. Forwarding headers are +// never consulted: the gate is meant to face callers directly, and a header a +// caller sets itself is not evidence of where it is. +func remoteAddr(r *http.Request) (netip.Addr, bool) { + ap, err := netip.ParseAddrPort(r.RemoteAddr) + if err != nil { + return netip.Addr{}, false + } + return ap.Addr().Unmap(), true +} + +func anyPrefixContains(prefixes []netip.Prefix, ip netip.Addr) bool { + for _, p := range prefixes { + if p.Contains(ip) { + return true + } + } + return false +} + +// refreshLocked re-reads the key file when its metadata changed since the last +// look. Caller holds g.mu. +func (g *Gate) refreshLocked() { + if g.filePath == "" || g.broken { + return + } + info, err := os.Stat(g.filePath) + var stamp fileStamp + switch { + case err == nil: + stamp = fileStamp{size: info.Size(), modTime: info.ModTime(), mode: info.Mode(), exists: true} + case errors.Is(err, fs.ErrNotExist): + stamp = fileStamp{} + default: + // A stat failure other than absence (a parent directory's permissions, + // an I/O error) counts as absence for this request and is re-examined + // on the next; report it when it is news. + if !g.fileChecked || g.fileStamp.exists { + slog.Error("authenticated LAN ingress: cannot stat key file; no file keys are in effect", + "path", g.filePath, "err", err) + } + stamp = fileStamp{} + } + if g.fileChecked && stamp == g.fileStamp { + return + } + g.fileChecked = true + g.fileStamp = stamp + g.fileKeys = nil + + if !stamp.exists { + if g.explicitFile { + slog.Error("authenticated LAN ingress: key file does not exist; no file keys are in effect", + "env", EnvKeysFile, "path", g.filePath) + } + return + } + keys, err := loadKeyFile(g.filePath, info) + if err != nil { + slog.Error("authenticated LAN ingress: key file ignored; no file keys are in effect", + "path", g.filePath, "err", err) + return + } + g.fileKeys = keys +} + +// loadKeyFile reads and validates a key file. On Unix-like systems the file must +// not be readable or writable by group or others; on Windows the mode bits +// carry no such meaning and the check is skipped. +func loadKeyFile(path string, info fs.FileInfo) ([]digest, error) { + if !info.Mode().IsRegular() { + return nil, errors.New("not a regular file") + } + if runtime.GOOS != "windows" { + if perm := info.Mode().Perm(); perm&0o077 != 0 { + return nil, fmt.Errorf("permissions %04o allow other users to read it; chmod 600", perm) + } + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + return parseKeys(f) +} + +// parseKeys reads one key per line. Blank lines and lines starting with '#' are +// ignored; surrounding whitespace, including a CR from a Windows editor, is +// trimmed. Any invalid entry fails the whole file: a key that can never match +// over the wire is a misconfiguration to surface, not to skip. +func parseKeys(r io.Reader) ([]digest, error) { + var keys []digest + sc := bufio.NewScanner(r) + line := 0 + for sc.Scan() { + line++ + entry := strings.TrimSpace(sc.Text()) + if entry == "" || strings.HasPrefix(entry, "#") { + continue + } + if err := validateKey(entry); err != nil { + return nil, fmt.Errorf("line %d: %w", line, err) + } + keys = append(keys, sha256.Sum256([]byte(entry))) + } + if err := sc.Err(); err != nil { + return nil, err + } + if len(keys) == 0 { + return nil, errors.New("contains no keys") + } + return keys, nil +} + +// validateKey enforces the shape a key must have to be usable at all: long +// enough to resist guessing, and printable ASCII with no whitespace so it +// survives an HTTP header unchanged. +func validateKey(key string) error { + if len(key) < MinKeyLength { + return fmt.Errorf("key is %d characters; at least %d are required", len(key), MinKeyLength) + } + for _, c := range key { + if c > unicode.MaxASCII || c <= ' ' || c == 0x7f { + return errors.New("key must be printable ASCII with no whitespace") + } + } + return nil +} + +// announceLocked logs a change in the gate's state — enabled with N keys, or +// back to disabled — once per change. Enabling is logged at Warn: it widens the +// proxy's exposure and an operator reading the log should see it plainly. +// Caller holds g.mu. +func (g *Gate) announceLocked() { + enabled := g.enabledLocked() + n := len(g.inline) + len(g.fileKeys) + if g.announcedOnce && enabled == g.announcedEnabled && n == g.announcedKeys { + return + } + g.announcedOnce, g.announcedEnabled, g.announcedKeys = true, enabled, n + if enabled { + cidrs := make([]string, 0, len(g.cidrs)) + for _, p := range g.cidrs { + cidrs = append(cidrs, p.String()) + } + slog.Warn("authenticated LAN ingress ENABLED: a non-loopback plaintext caller presenting a configured API key is routed", + "keys", n, "key_file", g.filePath, "allowed_cidrs", cidrs) + return + } + slog.Info("authenticated LAN ingress disabled; plaintext requests are accepted from loopback only", + "key_file", g.filePath) +} diff --git a/services/shared/ingressauth/ingressauth_test.go b/services/shared/ingressauth/ingressauth_test.go new file mode 100644 index 00000000..c7e2a70d --- /dev/null +++ b/services/shared/ingressauth/ingressauth_test.go @@ -0,0 +1,456 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package ingressauth + +import ( + "net/http" + "net/http/httptest" + "net/netip" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +const ( + keyA = "0123456789abcdef0123456789abcdef" // exactly MinKeyLength + keyB = "b-key-b-key-b-key-b-key-b-key-b-key-b-key-0000" // longer, with dashes + keyC = "sk-nvpair-cccccccccccccccccccccccccccccccccccccccccccccccccccccc" // prefixed, like SDK keys +) + +func request(remote string, hdr ...string) *http.Request { + r := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) + r.RemoteAddr = remote + for i := 0; i+1 < len(hdr); i += 2 { + r.Header.Set(hdr[i], hdr[i+1]) + } + return r +} + +func TestValidateKey(t *testing.T) { + cases := []struct { + name string + key string + ok bool + }{ + {"exactly minimum length", keyA, true}, + {"longer with punctuation", keyC, true}, + {"one short", keyA[:MinKeyLength-1], false}, + {"empty", "", false}, + {"embedded space", "0123456789abcdef 123456789abcdef0", false}, + {"embedded tab", "0123456789abcdef\t123456789abcdef0", false}, + {"non-ascii", "0123456789abcdef0123456789abcdé", false}, + {"control char", "0123456789abcdef0123456789abcde\x01", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := validateKey(tc.key) + if (err == nil) != tc.ok { + t.Fatalf("validateKey(%q) err = %v, want ok=%v", tc.key, err, tc.ok) + } + }) + } +} + +func TestParseKeysSkipsCommentsBlanksAndCRLF(t *testing.T) { + in := "# leading comment\r\n\r\n " + keyA + " \r\n" + keyB + "\n\n # trailing comment\n" + keys, err := parseKeys(strings.NewReader(in)) + if err != nil { + t.Fatalf("parseKeys: %v", err) + } + if len(keys) != 2 { + t.Fatalf("parsed %d keys, want 2", len(keys)) + } + g := &Gate{inline: keys} + for _, k := range []string{keyA, keyB} { + if d := g.Authorize(request("192.0.2.9:1", "Authorization", "Bearer "+k)); !d.Allowed { + t.Errorf("key %q from file not accepted: %+v", k, d) + } + } +} + +func TestParseKeysRejectsWholeFileOnOneBadEntry(t *testing.T) { + cases := map[string]string{ + "short entry": keyA + "\nshort\n", + "whitespace": "0123456789abcdef 123456789abcdef0\n", + "only comments": "# nothing here\n\n", + "empty": "", + "non-ascii line": keyA + "\n0123456789abcdef0123456789abcdé\n", + } + for name, in := range cases { + t.Run(name, func(t *testing.T) { + if keys, err := parseKeys(strings.NewReader(in)); err == nil { + t.Fatalf("parseKeys accepted %d keys, want an error", len(keys)) + } + }) + } +} + +func TestAuthorizeCredentials(t *testing.T) { + g := New([]string{keyA, keyB}, nil) + cases := []struct { + name string + hdr []string + allow bool + status int + fp string + }{ + {"bearer first key", []string{"Authorization", "Bearer " + keyA}, true, http.StatusOK, Fingerprint(keyA)}, + {"bearer second key", []string{"Authorization", "Bearer " + keyB}, true, http.StatusOK, Fingerprint(keyB)}, + {"lowercase scheme", []string{"Authorization", "bearer " + keyA}, true, http.StatusOK, Fingerprint(keyA)}, + {"x-api-key", []string{"X-Api-Key", keyA}, true, http.StatusOK, Fingerprint(keyA)}, + {"x-api-key lowercase header name", []string{"x-api-key", keyB}, true, http.StatusOK, Fingerprint(keyB)}, + {"no credential", nil, false, http.StatusUnauthorized, noCredential}, + {"wrong key", []string{"Authorization", "Bearer " + keyC}, false, http.StatusUnauthorized, Fingerprint(keyC)}, + {"wrong scheme", []string{"Authorization", "Basic " + keyA}, false, http.StatusUnauthorized, noCredential}, + {"bearer with no token", []string{"Authorization", "Bearer "}, false, http.StatusUnauthorized, noCredential}, + {"key as prefix only", []string{"Authorization", "Bearer " + keyA + "x"}, false, http.StatusUnauthorized, Fingerprint(keyA + "x")}, + {"wrong bearer but right x-api-key", []string{"Authorization", "Bearer " + keyC, "X-Api-Key", keyA}, false, http.StatusUnauthorized, Fingerprint(keyC)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + d := g.Authorize(request("192.0.2.9:40000", tc.hdr...)) + if d.Allowed != tc.allow || d.Status != tc.status { + t.Fatalf("decision = %+v, want allowed=%v status=%d", d, tc.allow, tc.status) + } + if d.KeyFingerprint != tc.fp { + t.Errorf("fingerprint = %q, want %q", d.KeyFingerprint, tc.fp) + } + if !tc.allow && d.Code != CodeUnauthorized { + t.Errorf("code = %q, want %q", d.Code, CodeUnauthorized) + } + if strings.Contains(d.Message, keyA) || strings.Contains(d.Message, keyC) { + t.Errorf("message echoes a key: %q", d.Message) + } + }) + } +} + +func TestAuthorizeCIDRAllowlist(t *testing.T) { + g := New([]string{keyA}, []netip.Prefix{ + netip.MustParsePrefix("192.168.1.0/24"), + netip.MustParsePrefix("fd00::/8"), + }) + auth := []string{"Authorization", "Bearer " + keyA} + cases := []struct { + name string + remote string + hdr []string + allow bool + code string + }{ + {"inside v4 with key", "192.168.1.77:5000", auth, true, ""}, + {"inside v6 with key", "[fd00::1]:5000", auth, true, ""}, + {"ipv4-mapped v6 inside", "[::ffff:192.168.1.77]:5000", auth, true, ""}, + {"outside with valid key", "192.168.2.77:5000", auth, false, CodeSourceNotAllowed}, + {"outside without key", "10.0.0.5:5000", nil, false, CodeSourceNotAllowed}, + {"inside without key", "192.168.1.77:5000", nil, false, CodeUnauthorized}, + {"unparseable remote", "garbage", auth, false, CodeSourceNotAllowed}, + {"empty remote", "", auth, false, CodeSourceNotAllowed}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + d := g.Authorize(request(tc.remote, tc.hdr...)) + if d.Allowed != tc.allow { + t.Fatalf("decision = %+v, want allowed=%v", d, tc.allow) + } + if !tc.allow && d.Code != tc.code { + t.Errorf("code = %q, want %q", d.Code, tc.code) + } + if d.Code == CodeSourceNotAllowed { + if d.Status != http.StatusForbidden { + t.Errorf("status = %d, want 403", d.Status) + } + // An out-of-policy source learns nothing about its key. + if d.KeyFingerprint != noCredential { + t.Errorf("fingerprint = %q, want %q for a source rejection", d.KeyFingerprint, noCredential) + } + } + }) + } +} + +func TestAuthorizeIgnoresForwardingHeaders(t *testing.T) { + g := New([]string{keyA}, []netip.Prefix{netip.MustParsePrefix("192.168.1.0/24")}) + d := g.Authorize(request("10.9.9.9:1", + "Authorization", "Bearer "+keyA, + "X-Forwarded-For", "192.168.1.5", + "X-Real-IP", "192.168.1.5", + "Forwarded", "for=192.168.1.5")) + if d.Allowed { + t.Fatal("a forwarding header moved the caller inside the allowlist") + } +} + +func TestDisabledGateNeverAllows(t *testing.T) { + var zero Gate + if zero.Enabled() { + t.Fatal("zero Gate reports enabled") + } + if d := zero.Authorize(request("192.0.2.9:1", "Authorization", "Bearer "+keyA)); d.Allowed { + t.Fatalf("zero Gate allowed a request: %+v", d) + } + empty := New(nil, nil) + if empty.Enabled() { + t.Fatal("New(nil, nil) reports enabled") + } +} + +func TestFingerprintIsShortStableHexAndNotTheKey(t *testing.T) { + fp := Fingerprint(keyA) + if len(fp) != 8 { + t.Fatalf("fingerprint %q has length %d, want 8", fp, len(fp)) + } + if strings.ToLower(fp) != fp || strings.Trim(fp, "0123456789abcdef") != "" { + t.Fatalf("fingerprint %q is not lowercase hex", fp) + } + if fp != Fingerprint(keyA) { + t.Fatal("fingerprint is not stable") + } + if fp == Fingerprint(keyB) { + t.Fatal("distinct keys share a fingerprint") + } + if strings.Contains(keyA, fp) { + t.Fatal("fingerprint is a substring of the key") + } +} + +func TestStripCredentialRemovesBothHeaders(t *testing.T) { + g := New([]string{keyA}, nil) + r := request("192.0.2.9:1", "Authorization", "Bearer "+keyA, "X-Api-Key", keyA, "Content-Type", "application/json") + g.StripCredential(r.Header) + if r.Header.Get("Authorization") != "" || r.Header.Get("X-Api-Key") != "" { + t.Fatalf("credential headers survived: %v", r.Header) + } + if r.Header.Get("Content-Type") != "application/json" { + t.Fatal("an unrelated header was removed") + } +} + +func TestNewPanicsOnInvalidLiteralKey(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatal("New accepted an invalid literal key") + } + }() + New([]string{"short"}, nil) +} + +// writeKeyFile writes content with mode and pins the modification time so a +// rewrite is distinguishable from the previous version even on filesystems +// with coarse timestamps. +func writeKeyFile(t *testing.T, path, content string, mode os.FileMode, when time.Time) { + t.Helper() + if err := os.WriteFile(path, []byte(content), mode); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, mode); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(path, when, when); err != nil { + t.Fatal(err) + } +} + +func TestKeyFileLoadsRotatesAndFailsClosed(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "keys") + g := &Gate{filePath: path, explicitFile: true} + base := time.Now().Add(-time.Hour).Truncate(time.Second) + + if g.Enabled() { + t.Fatal("enabled before the key file exists") + } + + writeKeyFile(t, path, "# first key\n"+keyA+"\n", 0o600, base) + if !g.Enabled() { + t.Fatal("not enabled after the key file appeared") + } + if d := g.Authorize(request("192.0.2.9:1", "Authorization", "Bearer "+keyA)); !d.Allowed { + t.Fatalf("file key rejected: %+v", d) + } + + // Rotation: replace A with B without a restart. + writeKeyFile(t, path, keyB+"\n", 0o600, base.Add(10*time.Second)) + if d := g.Authorize(request("192.0.2.9:1", "Authorization", "Bearer "+keyA)); d.Allowed { + t.Fatal("rotated-out key still accepted") + } + if d := g.Authorize(request("192.0.2.9:1", "Authorization", "Bearer "+keyB)); !d.Allowed { + t.Fatalf("rotated-in key rejected: %+v", d) + } + + // A malformed rewrite contributes nothing: the LAN closes rather than + // staying open on the previous keys. + writeKeyFile(t, path, keyB+"\nshort\n", 0o600, base.Add(20*time.Second)) + if g.Enabled() { + t.Fatal("enabled on a malformed key file") + } + if d := g.Authorize(request("192.0.2.9:1", "Authorization", "Bearer "+keyB)); d.Allowed { + t.Fatal("previous keys survived a failed reload") + } + + // Repairing the file re-enables; removing it disables. + writeKeyFile(t, path, keyB+"\n", 0o600, base.Add(30*time.Second)) + if !g.Enabled() { + t.Fatal("not re-enabled after the file was repaired") + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if g.Enabled() { + t.Fatal("enabled after the key file was removed") + } +} + +func TestKeyFilePermissionsMustBePrivate(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX permission bits are not meaningful on Windows") + } + dir := t.TempDir() + path := filepath.Join(dir, "keys") + g := &Gate{filePath: path, explicitFile: true} + when := time.Now().Add(-time.Hour).Truncate(time.Second) + + writeKeyFile(t, path, keyA+"\n", 0o644, when) + if g.Enabled() { + t.Fatal("enabled on a world-readable key file") + } + // chmod alone changes neither size nor mtime; the gate must still notice. + if err := os.Chmod(path, 0o600); err != nil { + t.Fatal(err) + } + if !g.Enabled() { + t.Fatal("not enabled after permissions were tightened") + } + if err := os.Chmod(path, 0o640); err != nil { + t.Fatal(err) + } + if g.Enabled() { + t.Fatal("enabled on a group-readable key file") + } +} + +func TestKeyFileMustBeRegular(t *testing.T) { + dir := t.TempDir() + g := &Gate{filePath: dir, explicitFile: true} + if g.Enabled() { + t.Fatal("a directory was accepted as a key file") + } +} + +// clearEnv points every variable the gate reads, and every base directory +// appdir consults, at the test's own scratch space. +func clearEnv(t *testing.T) string { + t.Helper() + home := t.TempDir() + t.Setenv(EnvKeys, "") + t.Setenv(EnvKeysFile, "") + t.Setenv(EnvAllowedCIDRs, "") + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", home) + t.Setenv("LOCALAPPDATA", home) + t.Setenv("APPDATA", home) + return home +} + +func TestFromEnvUnsetIsDisabled(t *testing.T) { + clearEnv(t) + g := FromEnv() + if g.Enabled() { + t.Fatal("FromEnv with nothing configured is enabled") + } + if g.filePath == "" || filepath.Base(g.filePath) != DefaultKeyFileName { + t.Fatalf("default key file path = %q, want .../%s", g.filePath, DefaultKeyFileName) + } +} + +func TestFromEnvInlineKeysAndFileCombine(t *testing.T) { + home := clearEnv(t) + path := filepath.Join(home, "keys") + writeKeyFile(t, path, keyB+"\n", 0o600, time.Now().Add(-time.Hour)) + t.Setenv(EnvKeys, " "+keyA+" , ,"+keyC) + t.Setenv(EnvKeysFile, path) + + g := FromEnv() + if !g.Enabled() { + t.Fatal("not enabled") + } + for _, k := range []string{keyA, keyB, keyC} { + if d := g.Authorize(request("192.0.2.9:1", "X-Api-Key", k)); !d.Allowed { + t.Errorf("key %q rejected: %+v", k, d) + } + } +} + +func TestFromEnvDefaultFileInAppDir(t *testing.T) { + home := clearEnv(t) + dir, err := os.UserConfigDir() + if err != nil { + t.Skip("no user config dir:", err) + } + if !strings.HasPrefix(dir, home) { + t.Skipf("os.UserConfigDir()=%q is not under the test HOME %q on this platform", dir, home) + } + appDir := filepath.Join(dir, "Nvidia Corporation", "Personal AI Router") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + writeKeyFile(t, filepath.Join(appDir, DefaultKeyFileName), keyA+"\n", 0o600, time.Now().Add(-time.Hour)) + + g := FromEnv() + if !g.Enabled() { + t.Fatalf("default key file at %q not picked up", g.filePath) + } +} + +func TestFromEnvInvalidInlineKeyDisablesEverything(t *testing.T) { + home := clearEnv(t) + path := filepath.Join(home, "keys") + writeKeyFile(t, path, keyB+"\n", 0o600, time.Now().Add(-time.Hour)) + t.Setenv(EnvKeysFile, path) + t.Setenv(EnvKeys, keyA+",too-short") + + g := FromEnv() + if g.Enabled() { + t.Fatal("enabled despite an invalid inline key") + } + if d := g.Authorize(request("192.0.2.9:1", "X-Api-Key", keyB)); d.Allowed { + t.Fatal("file key accepted while the environment is misconfigured") + } +} + +func TestFromEnvInvalidCIDRDisablesEverything(t *testing.T) { + clearEnv(t) + t.Setenv(EnvKeys, keyA) + t.Setenv(EnvAllowedCIDRs, "192.168.1.0/24, not-a-cidr") + + g := FromEnv() + if g.Enabled() { + t.Fatal("enabled despite a malformed CIDR allowlist") + } +} + +func TestFromEnvCIDRsAreMaskedAndApplied(t *testing.T) { + clearEnv(t) + t.Setenv(EnvKeys, keyA) + t.Setenv(EnvAllowedCIDRs, "192.168.1.77/24") + + g := FromEnv() + if d := g.Authorize(request("192.168.1.1:1", "X-Api-Key", keyA)); !d.Allowed { + t.Fatalf("host bits in the prefix were not masked: %+v", d) + } + if d := g.Authorize(request("192.168.2.1:1", "X-Api-Key", keyA)); d.Allowed { + t.Fatal("caller outside the allowlist admitted") + } +} + +func TestFromEnvExplicitMissingFileIsDisabled(t *testing.T) { + home := clearEnv(t) + t.Setenv(EnvKeysFile, filepath.Join(home, "does-not-exist")) + if g := FromEnv(); g.Enabled() { + t.Fatal("enabled with a missing explicit key file") + } +} From 35f3c44ca9f2db1d1d97fe355e34b30129ac479b Mon Sep 17 00:00:00 2001 From: "Aaron K. Clark (CryptoJones)" Date: Sat, 5 Sep 2026 19:47:41 -0500 Subject: [PATCH 2/4] Accept authenticated non-loopback plaintext on the Ollama and LM Studio proxies when keys are configured With no key configured nothing changes: a non-loopback plaintext request is still refused with 403 loopback-only. Once the operator configures a key, handlePlain consults nvpair-shared/ingressauth for a non-loopback caller: a caller outside NVPAIR_PROXY_ALLOWED_CIDRS gets 403 source-not-allowed, a missing or unknown key gets 401 unauthorized with a Bearer challenge, and a caller presenting a configured key has the credential stripped and is routed through the same local router a loopback client uses. Loopback callers are never asked for a key, and an OPTIONS preflight is still answered ahead of the gate. Bump ollama-proxy to 0.27.0 and lmstudio-proxy to 0.17.0 (additive HTTP-visible behavior), and product/installer to 0.92.0 per VERSIONING. Closes #28. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01BwxtwuoRxP75PdR6NAmMS3 Signed-off-by: Aaron K. Clark (CryptoJones) --- services/lmstudio-proxy/ingress.go | 46 +++- services/lmstudio-proxy/ingress_auth_test.go | 242 +++++++++++++++++++ services/lmstudio-proxy/main.go | 5 + services/lmstudio-proxy/proxy.go | 6 + services/ollama-proxy/ingress.go | 46 +++- services/ollama-proxy/ingress_auth_test.go | 242 +++++++++++++++++++ services/ollama-proxy/main.go | 5 + services/ollama-proxy/proxy.go | 6 + services/versions.json | 8 +- 9 files changed, 576 insertions(+), 30 deletions(-) create mode 100644 services/lmstudio-proxy/ingress_auth_test.go create mode 100644 services/ollama-proxy/ingress_auth_test.go diff --git a/services/lmstudio-proxy/ingress.go b/services/lmstudio-proxy/ingress.go index 2b70e697..d07ad772 100644 --- a/services/lmstudio-proxy/ingress.go +++ b/services/lmstudio-proxy/ingress.go @@ -55,25 +55,45 @@ func (p *Proxy) localBackendTarget() (*url.URL, bool) { return &url.URL{Scheme: "http", Host: net.JoinHostPort(host, strconv.Itoa(b.Port))}, true } -// handlePlain is the plaintext personality: it accepts requests only from -// loopback and hands them to the full local router (handleHTTP). A non-loopback -// caller — any LAN peer — is refused; peers must use the mTLS ingress. This is -// what closes the former open-relay exposure (the listener still binds all -// interfaces for the TLS personality, but plaintext is loopback-only). +// handlePlain is the plaintext personality: it accepts requests from loopback +// and hands them to the full local router (handleHTTP). A non-loopback caller — +// any LAN peer — is refused unless the operator has enabled the API-key gate +// (nvpair-shared/ingressauth) and the caller presents a configured key, in +// which case it is routed exactly like a loopback client. Cluster peers still +// use the mTLS ingress, and loopback is never asked for a key. This is what +// closes the former open-relay exposure: the listener binds all interfaces for +// the TLS personality, but plaintext is loopback-only unless authenticated. func (p *Proxy) handlePlain(w http.ResponseWriter, r *http.Request) { if !isLoopbackRemote(r.RemoteAddr) { // Answer a non-loopback preflight ahead of the gate. It grants no access - // on its own; the request that follows still receives the real 403. A - // loopback preflight continues into handleHTTP so an available engine's - // exact origin and credentials policy can be preserved. + // on its own; the request that follows still receives the real 401/403, + // and a browser sends no Authorization on a preflight anyway. A loopback + // preflight continues into handleHTTP so an available engine's exact + // origin and credentials policy can be preserved. if cors.WritePreflight(w, r) { return } - slog.Warn("rejected non-loopback plaintext request; cluster peers must use mTLS", - "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path) - writeIngressError(w, http.StatusForbidden, "loopback-only", - "plaintext requests are accepted only from loopback; cluster peers must use the mTLS ingress") - return + if p.lanAuth == nil || !p.lanAuth.Enabled() { + slog.Warn("rejected non-loopback plaintext request; cluster peers must use mTLS", + "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path) + writeIngressError(w, http.StatusForbidden, "loopback-only", + "plaintext requests are accepted only from loopback; cluster peers must use the mTLS ingress") + return + } + d := p.lanAuth.Authorize(r) + if !d.Allowed { + slog.Warn("rejected non-loopback plaintext request", "remote", r.RemoteAddr, + "method", r.Method, "path", r.URL.Path, "code", d.Code, "key_fp", d.KeyFingerprint) + if d.Status == http.StatusUnauthorized { + w.Header().Set("WWW-Authenticate", `Bearer realm="nvpair-proxy"`) + } + writeIngressError(w, d.Status, d.Code, d.Message) + return + } + // The key is the proxy's credential, not the engine's: never forward it. + p.lanAuth.StripCredential(r.Header) + slog.Debug("authenticated non-loopback plaintext request", "remote", r.RemoteAddr, + "method", r.Method, "path", r.URL.Path, "key_fp", d.KeyFingerprint) } // Engine-manager marks identity/action requests so the federated model-list // facade can never satisfy LM Studio's own /v1/models readiness probe. diff --git a/services/lmstudio-proxy/ingress_auth_test.go b/services/lmstudio-proxy/ingress_auth_test.go new file mode 100644 index 00000000..36b8f43f --- /dev/null +++ b/services/lmstudio-proxy/ingress_auth_test.go @@ -0,0 +1,242 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "sync/atomic" + "testing" + + "nvpair-shared/ingressauth" +) + +const ( + lanKey = "0123456789abcdef0123456789abcdef" + lanRemote = "192.0.2.50:40000" +) + +// authEngine is an httptest engine that records the headers of the last request +// it served, so a test can prove what the proxy did and did not forward. +func authEngine(t *testing.T) (*httptest.Server, *atomic.Pointer[http.Header]) { + t.Helper() + var seen atomic.Pointer[http.Header] + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h := r.Header.Clone() + seen.Store(&h) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(srv.Close) + return srv, &seen +} + +// lanProxy is a proxy with one routable engine and the API-key gate enabled for +// lanKey, optionally restricted to cidrs. +func lanProxy(t *testing.T, cidrs ...netip.Prefix) (*Proxy, *atomic.Pointer[http.Header]) { + t.Helper() + engine, seen := authEngine(t) + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "engine", engine.URL, "llama")) + p := testProxy(disc, 1235) + p.lanAuth = ingressauth.New([]string{lanKey}, cidrs) + return p, seen +} + +func inferenceRequest(remote string, hdr ...string) *http.Request { + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"llama","messages":[{"role":"user","content":"hi"}]}`)) + req.Header.Set("Content-Type", "application/json") + req.RemoteAddr = remote + for i := 0; i+1 < len(hdr); i += 2 { + req.Header.Set(hdr[i], hdr[i+1]) + } + return req +} + +func ingressCode(t *testing.T, rec *httptest.ResponseRecorder) string { + t.Helper() + var body struct { + Code string `json:"code"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("ingress error body %q is not JSON: %v", rec.Body.String(), err) + } + return body.Code +} + +// TestHandlePlainAuthenticatedNonLoopbackIsRouted: with the gate enabled, a LAN +// caller presenting a configured key is routed through the full local router +// like a loopback client, and the key is stripped before the engine sees it. +func TestHandlePlainAuthenticatedNonLoopbackIsRouted(t *testing.T) { + p, seen := lanProxy(t) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest(lanRemote, "Authorization", "Bearer "+lanKey)) + + if rec.Code != http.StatusOK { + t.Fatalf("authenticated LAN status = %d, want 200; body %s", rec.Code, rec.Body.String()) + } + h := seen.Load() + if h == nil { + t.Fatal("engine never received the routed request") + } + if got := h.Get("Authorization"); got != "" { + t.Errorf("engine received Authorization = %q, want the proxy's key stripped", got) + } + if got := h.Get("X-Api-Key"); got != "" { + t.Errorf("engine received X-Api-Key = %q, want stripped", got) + } +} + +func TestHandlePlainXApiKeyAccepted(t *testing.T) { + p, seen := lanProxy(t) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest(lanRemote, "X-Api-Key", lanKey)) + + if rec.Code != http.StatusOK { + t.Fatalf("X-Api-Key status = %d, want 200; body %s", rec.Code, rec.Body.String()) + } + if h := seen.Load(); h == nil || h.Get("X-Api-Key") != "" { + t.Fatalf("engine headers = %v, want the request forwarded without X-Api-Key", h) + } +} + +// TestHandlePlainNonLoopbackWithoutKeyIs401: an enabled gate turns the LAN +// refusal from 403 loopback-only into 401 with a challenge, still carrying CORS +// so a browser can read it, and nothing is forwarded. +func TestHandlePlainNonLoopbackWithoutKeyIs401(t *testing.T) { + p, seen := lanProxy(t) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest(lanRemote)) + + if rec.Code != http.StatusUnauthorized { + t.Fatalf("no-key LAN status = %d, want 401", rec.Code) + } + if got := ingressCode(t, rec); got != ingressauth.CodeUnauthorized { + t.Errorf("code = %q, want %q", got, ingressauth.CodeUnauthorized) + } + if got := rec.Header().Get("WWW-Authenticate"); got != `Bearer realm="nvpair-proxy"` { + t.Errorf("WWW-Authenticate = %q, want a Bearer challenge", got) + } + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" { + t.Errorf("Access-Control-Allow-Origin = %q, want * on the refusal", got) + } + if strings.Contains(rec.Body.String(), lanKey) { + t.Error("refusal body echoes a key") + } + if seen.Load() != nil { + t.Fatal("an unauthenticated LAN request reached the engine") + } +} + +func TestHandlePlainNonLoopbackWrongKeyIs401(t *testing.T) { + p, seen := lanProxy(t) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest(lanRemote, "Authorization", "Bearer "+lanKey+"-not")) + + if rec.Code != http.StatusUnauthorized { + t.Fatalf("wrong-key LAN status = %d, want 401", rec.Code) + } + if got := ingressCode(t, rec); got != ingressauth.CodeUnauthorized { + t.Errorf("code = %q, want %q", got, ingressauth.CodeUnauthorized) + } + if seen.Load() != nil { + t.Fatal("a request with a wrong key reached the engine") + } +} + +// TestHandlePlainLoopbackNeedsNoKeyWhenEnabled: enabling the gate changes +// nothing for loopback — no key is required, and a client's own Authorization +// header (an SDK placeholder, say) is forwarded untouched as it is today. +func TestHandlePlainLoopbackNeedsNoKeyWhenEnabled(t *testing.T) { + p, seen := lanProxy(t) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest("127.0.0.1:40000", "Authorization", "Bearer lm-studio")) + + if rec.Code != http.StatusOK { + t.Fatalf("loopback status = %d, want 200; body %s", rec.Code, rec.Body.String()) + } + h := seen.Load() + if h == nil { + t.Fatal("engine never received the loopback request") + } + if got := h.Get("Authorization"); got != "Bearer lm-studio" { + t.Errorf("loopback Authorization forwarded as %q, want it untouched", got) + } +} + +// TestHandlePlainOutsideAllowedCIDRIs403: with an allowlist configured, a +// caller outside it is refused before its key is examined — a valid key does not +// help, and no Bearer challenge is issued. +func TestHandlePlainOutsideAllowedCIDRIs403(t *testing.T) { + p, seen := lanProxy(t, netip.MustParsePrefix("10.0.0.0/8")) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest(lanRemote, "Authorization", "Bearer "+lanKey)) + + if rec.Code != http.StatusForbidden { + t.Fatalf("out-of-allowlist status = %d, want 403", rec.Code) + } + if got := ingressCode(t, rec); got != ingressauth.CodeSourceNotAllowed { + t.Errorf("code = %q, want %q", got, ingressauth.CodeSourceNotAllowed) + } + if got := rec.Header().Get("WWW-Authenticate"); got != "" { + t.Errorf("WWW-Authenticate = %q, want none on a source refusal", got) + } + if seen.Load() != nil { + t.Fatal("a request from outside the allowlist reached the engine") + } +} + +func TestHandlePlainInsideAllowedCIDRIsRouted(t *testing.T) { + p, seen := lanProxy(t, netip.MustParsePrefix("192.0.2.0/24")) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest(lanRemote, "Authorization", "Bearer "+lanKey)) + + if rec.Code != http.StatusOK { + t.Fatalf("in-allowlist status = %d, want 200; body %s", rec.Code, rec.Body.String()) + } + if seen.Load() == nil { + t.Fatal("engine never received the request") + } +} + +// TestHandlePlainPreflightStillAnsweredWhenEnabled: a browser sends no +// Authorization on a preflight, so the 204 must keep preceding the gate. +func TestHandlePlainPreflightStillAnsweredWhenEnabled(t *testing.T) { + p, seen := lanProxy(t) + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodOptions, "/v1/chat/completions", nil) + req.RemoteAddr = lanRemote + req.Header.Set("Access-Control-Request-Headers", "Authorization") + p.handlePlain(rec, req) + + if rec.Code != http.StatusNoContent { + t.Fatalf("LAN preflight status = %d, want 204", rec.Code) + } + if seen.Load() != nil { + t.Fatal("a preflight reached the engine") + } +} + +// TestHandlePlainGateWithoutKeysKeepsLoopbackOnly: a gate that exists but has no +// keys is the default: the LAN refusal stays the original 403 loopback-only. +func TestHandlePlainGateWithoutKeysKeepsLoopbackOnly(t *testing.T) { + p, seen := lanProxy(t) + p.lanAuth = ingressauth.New(nil, nil) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest(lanRemote, "Authorization", "Bearer "+lanKey)) + + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403", rec.Code) + } + if got := ingressCode(t, rec); got != "loopback-only" { + t.Errorf("code = %q, want loopback-only", got) + } + if seen.Load() != nil { + t.Fatal("a LAN request reached the engine with no keys configured") + } +} diff --git a/services/lmstudio-proxy/main.go b/services/lmstudio-proxy/main.go index 846ac9cb..0b54100d 100644 --- a/services/lmstudio-proxy/main.go +++ b/services/lmstudio-proxy/main.go @@ -16,6 +16,7 @@ import ( "nvpair-shared/applog" "nvpair-shared/clustertrust" + "nvpair-shared/ingressauth" ) func main() { @@ -83,6 +84,10 @@ func main() { // presence: a left/removed node keeps its keypair by design, and would // otherwise keep logging cluster_ingress with no cluster peers to serve. proxy.mesh = clustertrust.Open(*clusterDir) + // The opt-in API-key gate for non-loopback plaintext callers. Configured + // from the environment (or the default key file); with nothing configured + // it stays disabled and plaintext remains loopback-only. + proxy.lanAuth = ingressauth.FromEnv() go proxy.mesh.Watch(ctx, func(clustered bool) { slog.Info("cluster inference ingress switched personality", "cluster_ingress", clustered) proxy.dropUnpinnedPeerTransports() diff --git a/services/lmstudio-proxy/proxy.go b/services/lmstudio-proxy/proxy.go index 6e619e77..edb0d354 100644 --- a/services/lmstudio-proxy/proxy.go +++ b/services/lmstudio-proxy/proxy.go @@ -29,6 +29,7 @@ import ( "nvpair-shared/clustertrust" "nvpair-shared/cors" "nvpair-shared/errors" + "nvpair-shared/ingressauth" "nvpair-shared/netmon" "nvpair-shared/netpick" "nvpair-shared/nodeactivity" @@ -335,6 +336,11 @@ type Proxy struct { // only loopback-plaintext local routing. Read-only after startup. mesh *clustertrust.Mesh + // lanAuth is the opt-in API-key gate for non-loopback plaintext callers + // (nvpair-shared/ingressauth). nil or disabled: plaintext is loopback-only. + // Set once at startup; the gate re-reads its own key file on demand. + lanAuth *ingressauth.Gate + // backendMu guards backend, the explicit loopback engine the cluster mTLS // ingress forwards to. The broker sets/clears it via node/set-local-backend; // it is never sourced from discovery, so an ingress request can only ever diff --git a/services/ollama-proxy/ingress.go b/services/ollama-proxy/ingress.go index 984b7b20..1d5e646b 100644 --- a/services/ollama-proxy/ingress.go +++ b/services/ollama-proxy/ingress.go @@ -55,25 +55,45 @@ func (p *Proxy) localBackendTarget() (*url.URL, bool) { return &url.URL{Scheme: "http", Host: net.JoinHostPort(host, strconv.Itoa(b.Port))}, true } -// handlePlain is the plaintext personality: it accepts requests only from -// loopback and hands them to the full local router (handleHTTP). A non-loopback -// caller — any LAN peer — is refused; peers must use the mTLS ingress. This is -// what closes the former open-relay exposure (the listener still binds all -// interfaces for the TLS personality, but plaintext is loopback-only). +// handlePlain is the plaintext personality: it accepts requests from loopback +// and hands them to the full local router (handleHTTP). A non-loopback caller — +// any LAN peer — is refused unless the operator has enabled the API-key gate +// (nvpair-shared/ingressauth) and the caller presents a configured key, in +// which case it is routed exactly like a loopback client. Cluster peers still +// use the mTLS ingress, and loopback is never asked for a key. This is what +// closes the former open-relay exposure: the listener binds all interfaces for +// the TLS personality, but plaintext is loopback-only unless authenticated. func (p *Proxy) handlePlain(w http.ResponseWriter, r *http.Request) { if !isLoopbackRemote(r.RemoteAddr) { // Answer a non-loopback preflight ahead of the gate. It grants no access - // on its own; the request that follows still receives the real 403. A - // loopback preflight continues into handleHTTP so an available engine's - // exact origin and credentials policy can be preserved. + // on its own; the request that follows still receives the real 401/403, + // and a browser sends no Authorization on a preflight anyway. A loopback + // preflight continues into handleHTTP so an available engine's exact + // origin and credentials policy can be preserved. if cors.WritePreflight(w, r) { return } - slog.Warn("rejected non-loopback plaintext request; cluster peers must use mTLS", - "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path) - writeIngressError(w, http.StatusForbidden, "loopback-only", - "plaintext requests are accepted only from loopback; cluster peers must use the mTLS ingress") - return + if p.lanAuth == nil || !p.lanAuth.Enabled() { + slog.Warn("rejected non-loopback plaintext request; cluster peers must use mTLS", + "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path) + writeIngressError(w, http.StatusForbidden, "loopback-only", + "plaintext requests are accepted only from loopback; cluster peers must use the mTLS ingress") + return + } + d := p.lanAuth.Authorize(r) + if !d.Allowed { + slog.Warn("rejected non-loopback plaintext request", "remote", r.RemoteAddr, + "method", r.Method, "path", r.URL.Path, "code", d.Code, "key_fp", d.KeyFingerprint) + if d.Status == http.StatusUnauthorized { + w.Header().Set("WWW-Authenticate", `Bearer realm="nvpair-proxy"`) + } + writeIngressError(w, d.Status, d.Code, d.Message) + return + } + // The key is the proxy's credential, not the engine's: never forward it. + p.lanAuth.StripCredential(r.Header) + slog.Debug("authenticated non-loopback plaintext request", "remote", r.RemoteAddr, + "method", r.Method, "path", r.URL.Path, "key_fp", d.KeyFingerprint) } // Engine-manager marks its private identity/action requests so this // compatibility facade can never be mistaken for the local Ollama backend. diff --git a/services/ollama-proxy/ingress_auth_test.go b/services/ollama-proxy/ingress_auth_test.go new file mode 100644 index 00000000..eb48b76c --- /dev/null +++ b/services/ollama-proxy/ingress_auth_test.go @@ -0,0 +1,242 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "sync/atomic" + "testing" + + "nvpair-shared/ingressauth" +) + +const ( + lanKey = "0123456789abcdef0123456789abcdef" + lanRemote = "192.0.2.50:40000" +) + +// authEngine is an httptest engine that records the headers of the last request +// it served, so a test can prove what the proxy did and did not forward. +func authEngine(t *testing.T) (*httptest.Server, *atomic.Pointer[http.Header]) { + t.Helper() + var seen atomic.Pointer[http.Header] + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h := r.Header.Clone() + seen.Store(&h) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(srv.Close) + return srv, &seen +} + +// lanProxy is a proxy with one routable engine and the API-key gate enabled for +// lanKey, optionally restricted to cidrs. +func lanProxy(t *testing.T, cidrs ...netip.Prefix) (*Proxy, *atomic.Pointer[http.Header]) { + t.Helper() + engine, seen := authEngine(t) + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "engine", engine.URL, "llama")) + p := testProxy(disc, 11435) + p.lanAuth = ingressauth.New([]string{lanKey}, cidrs) + return p, seen +} + +func inferenceRequest(remote string, hdr ...string) *http.Request { + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"llama","messages":[{"role":"user","content":"hi"}]}`)) + req.Header.Set("Content-Type", "application/json") + req.RemoteAddr = remote + for i := 0; i+1 < len(hdr); i += 2 { + req.Header.Set(hdr[i], hdr[i+1]) + } + return req +} + +func ingressCode(t *testing.T, rec *httptest.ResponseRecorder) string { + t.Helper() + var body struct { + Code string `json:"code"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("ingress error body %q is not JSON: %v", rec.Body.String(), err) + } + return body.Code +} + +// TestHandlePlainAuthenticatedNonLoopbackIsRouted: with the gate enabled, a LAN +// caller presenting a configured key is routed through the full local router +// like a loopback client, and the key is stripped before the engine sees it. +func TestHandlePlainAuthenticatedNonLoopbackIsRouted(t *testing.T) { + p, seen := lanProxy(t) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest(lanRemote, "Authorization", "Bearer "+lanKey)) + + if rec.Code != http.StatusOK { + t.Fatalf("authenticated LAN status = %d, want 200; body %s", rec.Code, rec.Body.String()) + } + h := seen.Load() + if h == nil { + t.Fatal("engine never received the routed request") + } + if got := h.Get("Authorization"); got != "" { + t.Errorf("engine received Authorization = %q, want the proxy's key stripped", got) + } + if got := h.Get("X-Api-Key"); got != "" { + t.Errorf("engine received X-Api-Key = %q, want stripped", got) + } +} + +func TestHandlePlainXApiKeyAccepted(t *testing.T) { + p, seen := lanProxy(t) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest(lanRemote, "X-Api-Key", lanKey)) + + if rec.Code != http.StatusOK { + t.Fatalf("X-Api-Key status = %d, want 200; body %s", rec.Code, rec.Body.String()) + } + if h := seen.Load(); h == nil || h.Get("X-Api-Key") != "" { + t.Fatalf("engine headers = %v, want the request forwarded without X-Api-Key", h) + } +} + +// TestHandlePlainNonLoopbackWithoutKeyIs401: an enabled gate turns the LAN +// refusal from 403 loopback-only into 401 with a challenge, still carrying CORS +// so a browser can read it, and nothing is forwarded. +func TestHandlePlainNonLoopbackWithoutKeyIs401(t *testing.T) { + p, seen := lanProxy(t) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest(lanRemote)) + + if rec.Code != http.StatusUnauthorized { + t.Fatalf("no-key LAN status = %d, want 401", rec.Code) + } + if got := ingressCode(t, rec); got != ingressauth.CodeUnauthorized { + t.Errorf("code = %q, want %q", got, ingressauth.CodeUnauthorized) + } + if got := rec.Header().Get("WWW-Authenticate"); got != `Bearer realm="nvpair-proxy"` { + t.Errorf("WWW-Authenticate = %q, want a Bearer challenge", got) + } + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" { + t.Errorf("Access-Control-Allow-Origin = %q, want * on the refusal", got) + } + if strings.Contains(rec.Body.String(), lanKey) { + t.Error("refusal body echoes a key") + } + if seen.Load() != nil { + t.Fatal("an unauthenticated LAN request reached the engine") + } +} + +func TestHandlePlainNonLoopbackWrongKeyIs401(t *testing.T) { + p, seen := lanProxy(t) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest(lanRemote, "Authorization", "Bearer "+lanKey+"-not")) + + if rec.Code != http.StatusUnauthorized { + t.Fatalf("wrong-key LAN status = %d, want 401", rec.Code) + } + if got := ingressCode(t, rec); got != ingressauth.CodeUnauthorized { + t.Errorf("code = %q, want %q", got, ingressauth.CodeUnauthorized) + } + if seen.Load() != nil { + t.Fatal("a request with a wrong key reached the engine") + } +} + +// TestHandlePlainLoopbackNeedsNoKeyWhenEnabled: enabling the gate changes +// nothing for loopback — no key is required, and a client's own Authorization +// header (an SDK placeholder, say) is forwarded untouched as it is today. +func TestHandlePlainLoopbackNeedsNoKeyWhenEnabled(t *testing.T) { + p, seen := lanProxy(t) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest("127.0.0.1:40000", "Authorization", "Bearer lm-studio")) + + if rec.Code != http.StatusOK { + t.Fatalf("loopback status = %d, want 200; body %s", rec.Code, rec.Body.String()) + } + h := seen.Load() + if h == nil { + t.Fatal("engine never received the loopback request") + } + if got := h.Get("Authorization"); got != "Bearer lm-studio" { + t.Errorf("loopback Authorization forwarded as %q, want it untouched", got) + } +} + +// TestHandlePlainOutsideAllowedCIDRIs403: with an allowlist configured, a +// caller outside it is refused before its key is examined — a valid key does not +// help, and no Bearer challenge is issued. +func TestHandlePlainOutsideAllowedCIDRIs403(t *testing.T) { + p, seen := lanProxy(t, netip.MustParsePrefix("10.0.0.0/8")) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest(lanRemote, "Authorization", "Bearer "+lanKey)) + + if rec.Code != http.StatusForbidden { + t.Fatalf("out-of-allowlist status = %d, want 403", rec.Code) + } + if got := ingressCode(t, rec); got != ingressauth.CodeSourceNotAllowed { + t.Errorf("code = %q, want %q", got, ingressauth.CodeSourceNotAllowed) + } + if got := rec.Header().Get("WWW-Authenticate"); got != "" { + t.Errorf("WWW-Authenticate = %q, want none on a source refusal", got) + } + if seen.Load() != nil { + t.Fatal("a request from outside the allowlist reached the engine") + } +} + +func TestHandlePlainInsideAllowedCIDRIsRouted(t *testing.T) { + p, seen := lanProxy(t, netip.MustParsePrefix("192.0.2.0/24")) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest(lanRemote, "Authorization", "Bearer "+lanKey)) + + if rec.Code != http.StatusOK { + t.Fatalf("in-allowlist status = %d, want 200; body %s", rec.Code, rec.Body.String()) + } + if seen.Load() == nil { + t.Fatal("engine never received the request") + } +} + +// TestHandlePlainPreflightStillAnsweredWhenEnabled: a browser sends no +// Authorization on a preflight, so the 204 must keep preceding the gate. +func TestHandlePlainPreflightStillAnsweredWhenEnabled(t *testing.T) { + p, seen := lanProxy(t) + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodOptions, "/v1/chat/completions", nil) + req.RemoteAddr = lanRemote + req.Header.Set("Access-Control-Request-Headers", "Authorization") + p.handlePlain(rec, req) + + if rec.Code != http.StatusNoContent { + t.Fatalf("LAN preflight status = %d, want 204", rec.Code) + } + if seen.Load() != nil { + t.Fatal("a preflight reached the engine") + } +} + +// TestHandlePlainGateWithoutKeysKeepsLoopbackOnly: a gate that exists but has no +// keys is the default: the LAN refusal stays the original 403 loopback-only. +func TestHandlePlainGateWithoutKeysKeepsLoopbackOnly(t *testing.T) { + p, seen := lanProxy(t) + p.lanAuth = ingressauth.New(nil, nil) + rec := httptest.NewRecorder() + p.handlePlain(rec, inferenceRequest(lanRemote, "Authorization", "Bearer "+lanKey)) + + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403", rec.Code) + } + if got := ingressCode(t, rec); got != "loopback-only" { + t.Errorf("code = %q, want loopback-only", got) + } + if seen.Load() != nil { + t.Fatal("a LAN request reached the engine with no keys configured") + } +} diff --git a/services/ollama-proxy/main.go b/services/ollama-proxy/main.go index afd147dd..f497bd16 100644 --- a/services/ollama-proxy/main.go +++ b/services/ollama-proxy/main.go @@ -17,6 +17,7 @@ import ( "nvpair-shared/applog" "nvpair-shared/clustertrust" + "nvpair-shared/ingressauth" ) type aliasAddressFlags []string @@ -99,6 +100,10 @@ func main() { // presence: a left/removed node keeps its keypair by design, and would // otherwise keep logging cluster_ingress with no cluster peers to serve. proxy.mesh = clustertrust.Open(*clusterDir) + // The opt-in API-key gate for non-loopback plaintext callers. Configured + // from the environment (or the default key file); with nothing configured + // it stays disabled and plaintext remains loopback-only. + proxy.lanAuth = ingressauth.FromEnv() go proxy.mesh.Watch(ctx, func(clustered bool) { slog.Info("cluster inference ingress switched personality", "cluster_ingress", clustered) diff --git a/services/ollama-proxy/proxy.go b/services/ollama-proxy/proxy.go index ad4ac7a2..866c2dee 100644 --- a/services/ollama-proxy/proxy.go +++ b/services/ollama-proxy/proxy.go @@ -30,6 +30,7 @@ import ( "nvpair-shared/clustertrust" "nvpair-shared/cors" "nvpair-shared/errors" + "nvpair-shared/ingressauth" "nvpair-shared/netmon" "nvpair-shared/netpick" "nvpair-shared/nodeactivity" @@ -344,6 +345,11 @@ type Proxy struct { // only loopback-plaintext local routing. Read-only after startup. mesh *clustertrust.Mesh + // lanAuth is the opt-in API-key gate for non-loopback plaintext callers + // (nvpair-shared/ingressauth). nil or disabled: plaintext is loopback-only. + // Set once at startup; the gate re-reads its own key file on demand. + lanAuth *ingressauth.Gate + // backendMu guards backend, the explicit loopback engine the cluster mTLS // ingress forwards to. The broker sets/clears it via node/set-local-backend; // it is never sourced from discovery, so an ingress request can only ever diff --git a/services/versions.json b/services/versions.json index 29d8c230..b6d9b40e 100644 --- a/services/versions.json +++ b/services/versions.json @@ -1,10 +1,10 @@ { "$comment": "Single source of truth for all version numbers. See VERSIONING.md for bump rules.", - "product": "0.91.7", - "installer": "0.91.7", + "product": "0.92.0", + "installer": "0.92.0", "components": { - "ollama-proxy": "0.26.2", - "lmstudio-proxy": "0.16.2", + "ollama-proxy": "0.27.0", + "lmstudio-proxy": "0.17.0", "nvpair-node-info": "0.13.3", "nvpair-node-scanner": "0.20.3", "nvpair-manual-nodes": "0.11.1", From dc8d8a953fbebebff19f5bbf7eee2fcf78517215 Mon Sep 17 00:00:00 2001 From: "Aaron K. Clark (CryptoJones)" Date: Sat, 5 Sep 2026 19:47:41 -0500 Subject: [PATCH 3/4] Document opt-in authenticated LAN access to the compatibility endpoints Loopback-only stays the documented default. SECURITY.md, the architecture trust tables, the getting-started guide (a new "Reaching PAIR from Another Machine" section with key generation, file location per OS, permissions, and client configuration), troubleshooting (the 401 unauthorized and 403 source-not-allowed cases), the overview, both proxy READMEs (environment variables and response codes), and the OpenAPI description (securitySchemes) now describe the opt-in gate, what it checks, what it does not add, and the exposure an operator accepts. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01BwxtwuoRxP75PdR6NAmMS3 Signed-off-by: Aaron K. Clark (CryptoJones) --- SECURITY.md | 31 ++++++++++--- docs/architecture.mdx | 21 ++++++--- docs/getting-started.mdx | 74 ++++++++++++++++++++++++++----- docs/overview.mdx | 6 ++- docs/troubleshooting.mdx | 15 +++++++ fern/openapi.yml | 17 +++++++ services/lmstudio-proxy/README.md | 13 +++++- services/ollama-proxy/README.md | 13 +++++- 8 files changed, 164 insertions(+), 26 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 957f9da4..e8548cbf 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -64,17 +64,38 @@ transport. These statements describe security boundaries visible in the current source. They are not claims that every deployment is secure. -### Inference Endpoints Are Loopback-Only +### Inference Endpoints Are Loopback-Only by Default A proxy's plaintext personality accepts requests from loopback only. It refuses a plaintext request from any other address. The same port also serves a mutual-TLS ingress for paired cluster members, and PAIR forwards that traffic to the node's own engine rather than routing it onward. -This is deliberate. A network-reachable plaintext endpoint would make any node an -open relay for inference to anything that can route to it. Run an application on -a node and use that node's local endpoint. Exposing an engine to the network -directly is outside PAIR and is the operator's decision and risk. +This is deliberate. A network-reachable plaintext endpoint without a credential +would make any node an open relay for inference to anything that can route to +it. Run an application on a node and use that node's local endpoint. Exposing an +engine to the network directly is outside PAIR and is the operator's decision +and risk. + +An operator can opt in to authenticated LAN access by configuring one or more +API keys (`NVPAIR_PROXY_API_KEYS_FILE`, default `proxy-api-keys` in PAIR's data +directory, or `NVPAIR_PROXY_API_KEYS`). With a key configured, a non-loopback +plaintext request is admitted only when it presents a configured key as +`Authorization: Bearer ` or `X-Api-Key: `, and, when +`NVPAIR_PROXY_ALLOWED_CIDRS` is set, only from a listed source range. An admitted +request is routed like a loopback client, and the key is stripped before the +request is forwarded, so it never reaches an engine or a peer. Loopback callers +are never asked for a key. + +The gate compares keys in constant time, holds them in memory only as digests, +never logs a presented key (a rejection logs a short digest fingerprint), and +fails closed: a key file that other users can read, that contains a malformed +entry, or that cannot be read contributes no keys and the LAN stays closed. +Enabling the gate is logged at warning level at startup and whenever the key set +changes. It adds no TLS, rate limiting, or per-key permissions: the plaintext +personality stays plaintext, so use it only on a network you trust or behind a +TLS-terminating proxy you control, and treat a configured key as a credential +that grants everything a local application can do. ### Local Network Is a Trust-Relevant Boundary diff --git a/docs/architecture.mdx b/docs/architecture.mdx index f7e7ffed..e7ede0fc 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -166,7 +166,7 @@ Not every surface is gated, and the exceptions are deliberate: | Surface | Transport | | --- | --- | -| Proxy, local clients | Plaintext HTTP, loopback only | +| Proxy, local clients | Plaintext HTTP, loopback only (opt-in: API-key authenticated LAN callers) | | Proxy, cluster ingress | Mutual TLS | | Model inventory, remote engine control | Mutual TLS | | Workload replication, error synchronization | Mutual TLS | @@ -192,7 +192,7 @@ port, chosen by the connection's first byte. A TLS handshake record starts with | First Byte | Personality | Who Uses It | | --- | --- | --- | -| Not `0x16` | Plaintext HTTP, loopback only | Your local applications | +| Not `0x16` | Plaintext HTTP, loopback only by default | Your local applications; API-key holders when the operator enables LAN access | | `0x16` | Mutual TLS | Paired nodes in the cluster | This is why an endpoint is `http://127.0.0.1:11434` for an application on the @@ -203,15 +203,22 @@ plaintext. The loopback restriction is enforced, not merely conventional. The listener binds all interfaces so the TLS personality can accept peers, but a plaintext request from any non-loopback address is refused with `403`. Without that check the port -would be an open relay for anything on the network. +would be an open relay for anything on the network. The one exception is opt-in: +an operator who configures API keys (refer to +[Reaching PAIR from Another Machine](getting-started.mdx#reaching-pair-from-another-machine)) +admits a non-loopback caller that presents a configured key to the same router a +loopback client uses. A caller without one is still refused, with `401`, and +loopback callers are never asked for a key. Two consequences follow, and together they define how clients are expected to reach PAIR: -- **A machine that is not a node has no way in.** It cannot use the plaintext - personality, because it is not loopback, and it cannot use the TLS personality, - because it holds no pinned cluster certificate. Pointing an application at - another machine's proxy port does not work by design. +- **A machine that is not a node has no way in unless the operator hands it a + key.** It cannot use the plaintext personality, because it is not loopback, and + it cannot use the TLS personality, because it holds no pinned cluster + certificate. Pointing an application at another machine's proxy port does not + work by design, until that machine's operator enables authenticated LAN access + and gives the application an API key. - **A peer request is served, not re-routed.** The mTLS ingress forwards straight to that node's own engine and never re-enters candidate selection, so a peer cannot chain a request onward through a third node. diff --git a/docs/getting-started.mdx b/docs/getting-started.mdx index cb7e7e34..c89cf629 100644 --- a/docs/getting-started.mdx +++ b/docs/getting-started.mdx @@ -427,11 +427,12 @@ PAIR. This is the part that surprises people, so it is worth stating directly. -**An endpoint only accepts requests from the machine it is on.** A PAIR proxy -serves plaintext HTTP to loopback only. PAIR refuses a request arriving from -anywhere else on the network with `403`, and the message says so. You cannot -point an application on a fourth machine at `http://some-node:11434` and have it -work. +**By default, an endpoint only accepts requests from the machine it is on.** A +PAIR proxy serves plaintext HTTP to loopback only. PAIR refuses a request +arriving from anywhere else on the network with `403`, and the message says so. +You cannot point an application on a fourth machine at `http://some-node:11434` +and have it work, unless that node's operator has enabled authenticated access, +described below. **Run PAIR where you work.** The intended pattern is to install PAIR on the machine you use, pair it into the cluster, and point your applications at *its* @@ -443,11 +444,64 @@ PAIR makes the routing decision where you make the request. When a cluster peer reaches a node over its authenticated channel, that node's own engine serves the request and does not forward it onward. -**If you need a network-reachable inference endpoint,** that is outside -what PAIR does. You would configure an engine to listen on your network yourself -and take on the exposure that implies. PAIR does not offer it by default and there -is no plan to add it as an option, because it would turn any node into an open -relay for anything on the network. +**If you need a network-reachable inference endpoint,** PAIR can provide one, +but only when you turn it on and only to clients that hold a key. PAIR never +offers it by default, because an endpoint anything on the network can use +without a credential would turn the node into an open relay. + +### Reaching PAIR from Another Machine + +Some clients cannot run PAIR themselves: an automation host, a container, a +Kubernetes workload, an SDK on a machine you do not administer. For those, a +node can accept requests from the network when the caller presents an API key. +The default is unchanged. With no key configured, the endpoint stays local. + +1. Generate a key. Anything of at least 32 printable ASCII characters with no + whitespace works; a random one is best: + + ```bash + openssl rand -hex 32 + ``` + +2. Put it in the proxy key file on the node, one key per line (`#` starts a + comment). The default location is `proxy-api-keys` in PAIR's data directory: + + | OS | Path | + | --- | --- | + | Windows | `%LOCALAPPDATA%\Nvidia Corporation\Personal AI Router\proxy-api-keys` | + | Linux | `~/.config/Nvidia Corporation/Personal AI Router/proxy-api-keys` | + | macOS | `~/Library/Application Support/Nvidia Corporation/Personal AI Router/proxy-api-keys` | + + On Linux and macOS the file must be readable by you alone (`chmod 600`). A + file other users can read is ignored, and the node's log says so. + +3. Point the client at the node's address and the port shown in **Endpoints**, + for example `http://gpu-box:11434` or `http://gpu-box:1234/v1`, and send the + key as a bearer token: + + ```bash + curl http://gpu-box:1234/v1/models -H "Authorization: Bearer $KEY" + ``` + + OpenAI-compatible SDKs send this header when you give them the key as their + API key. Clients built on the Anthropic convention may send + `X-Api-Key: ` instead. + +The proxy notices the key file appearing, changing, or disappearing on its own, +so you can add, rotate, or revoke keys without restarting PAIR. Three environment +variables refine the behavior for headless or containerized nodes: +`NVPAIR_PROXY_API_KEYS_FILE` names a different key file, `NVPAIR_PROXY_API_KEYS` +supplies keys inline (comma-separated), and `NVPAIR_PROXY_ALLOWED_CIDRS` (for +example `192.168.1.0/24,10.0.0.0/8`) additionally restricts which source networks +may use a key at all. + +Understand what you are enabling. The connection is plain HTTP, so prompts and +responses cross your network unencrypted, and anyone holding the key can use the +node, and through it every node in the cluster, exactly as a local application +could. Use it on a network you trust, keep the key private, and prefer the CIDR +allowlist. Applications on the node itself keep working over loopback without a +key. The node logs a warning when authenticated access is enabled and logs every +request it refuses, identifying a rejected key only by a short fingerprint. ## Verify It Is Working diff --git a/docs/overview.mdx b/docs/overview.mdx index 95ea262c..2f773a5a 100644 --- a/docs/overview.mdx +++ b/docs/overview.mdx @@ -112,7 +112,8 @@ and the cluster refuses a machine that is not a member. The PIN is a short convenience code for bootstrapping that exchange, not a strong authenticator, so pair only over networks and with machines you trust. Local -applications reach the proxy over loopback. +applications reach the proxy over loopback; an operator can also admit +applications on other machines by API key. For the trust boundaries in detail, refer to [Architecture](architecture.mdx) and the [security policy](../SECURITY.md). @@ -126,7 +127,8 @@ For the trust boundaries in detail, refer to - Model-aware, workload-informed routing of independent requests. - Encrypted routing between machines: a request sent to another node travels over mutual TLS restricted to the nodes you have paired, and a machine that is not a - cluster member is refused. Local applications reach the proxy over loopback. + cluster member is refused. Local applications reach the proxy over loopback, + and an operator can opt in to API-key access for applications on other machines. - A desktop application, plus a terminal interface for headless machines, driving the same services. - Visibility into nodes, engines, models, workloads, and service errors. diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index e4d8822c..ab44b5e7 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -98,6 +98,21 @@ needs no GPU or engine of its own, and its local endpoint routes to nodes that have them. Refer to [The Endpoint Is Local to the Machine Running PAIR](getting-started.mdx#the-endpoint-is-local-to-the-machine-running-pair). +If the application must stay where it is, the node's operator can enable +authenticated access with an API key; refer to +[Reaching PAIR from Another Machine](getting-started.mdx#reaching-pair-from-another-machine). +The refusal then says what is missing: + +- `403` `loopback-only`: the node has no usable key configured. Check the node's + log; a key file that other users can read, or that contains a malformed entry, + is ignored and the reason is logged. +- `401` `unauthorized`: the request carried no key, or a key the node does not + have. Send it as `Authorization: Bearer ` (or `X-Api-Key: `). The + node's log shows the first eight hex digits of the SHA-256 of the key it + received, so you can tell a missing key from a mistyped one. +- `403` `source-not-allowed`: the node restricts callers with + `NVPAIR_PROXY_ALLOWED_CIDRS` and the application's address is outside it. + For an application on the same machine, copy the URL from **Endpoints > API endpoints** rather than assuming a port. PAIR takes the engine's usual port for its compatible proxy and moves the engine itself to the diff --git a/fern/openapi.yml b/fern/openapi.yml index 2a9b2ac8..2fc0d088 100644 --- a/fern/openapi.yml +++ b/fern/openapi.yml @@ -195,6 +195,23 @@ components: application/json: schema: $ref: "#/components/schemas/Error" + securitySchemes: + bearerAuth: + type: http + scheme: bearer + description: > + Needed only by a caller that is not on the node itself, and only when + the node's operator has enabled authenticated LAN access by configuring + API keys. Loopback callers send no credential. Send a configured key as + `Authorization: Bearer `; the proxy strips it before forwarding. + apiKeyAuth: + type: apiKey + in: header + name: X-Api-Key + description: > + Alternative to bearerAuth for clients built on the Anthropic SDK + convention. Same scope: non-loopback callers, when the operator has + enabled LAN access. schemas: ChatRole: type: string diff --git a/services/lmstudio-proxy/README.md b/services/lmstudio-proxy/README.md index 71a8b70d..f3eb5fa4 100644 --- a/services/lmstudio-proxy/README.md +++ b/services/lmstudio-proxy/README.md @@ -36,7 +36,18 @@ lmstudio-proxy [flags] The proxy listens on `--port` (default 1234) and forwards incoming HTTP requests to the currently active LM Studio node — except the model-list route `GET /v1/models`, which is queried across every candidate node concurrently and merged into one de-duplicated inventory. Point your OpenAI-compatible client at `http://localhost:1234` and the proxy handles routing. -**Cluster ingress.** The listener carries two personalities, demultiplexed by each connection's first byte. Plaintext HTTP is accepted only from loopback; a LAN caller is refused. When `--cluster-dir` shows this node is a cluster member, the same listener also terminates cluster mTLS: a peer whose client certificate matches one of this node's pins is forwarded straight to the local engine reported by `node/set-local-backend`, and is never re-routed onward to another node. Membership and pins are re-derived per request, so joining or leaving a cluster needs no restart. +**Cluster ingress.** The listener carries two personalities, demultiplexed by each connection's first byte. Plaintext HTTP is accepted from loopback; a LAN caller is refused unless the operator has enabled authenticated LAN access (next paragraph). When `--cluster-dir` shows this node is a cluster member, the same listener also terminates cluster mTLS: a peer whose client certificate matches one of this node's pins is forwarded straight to the local engine reported by `node/set-local-backend`, and is never re-routed onward to another node. Membership and pins are re-derived per request, so joining or leaving a cluster needs no restart. + +**Authenticated LAN access (opt-in).** With no API key configured, a non-loopback plaintext request is refused with `403` `loopback-only`. An operator enables LAN access by configuring at least one key; the proxy then admits a non-loopback caller that presents a configured key as `Authorization: Bearer ` or `X-Api-Key: `, routes it exactly like a loopback client, and strips the key before forwarding so it never reaches an engine or a peer. Loopback callers are never asked for a key. The gate is `nvpair-shared/ingressauth`, shared with [`ollama-proxy`](../ollama-proxy/README.md) so both proxies enforce it identically. + +| Variable | Meaning | +|---|---| +| `NVPAIR_PROXY_API_KEYS_FILE` | Key file, one key per line; blank lines and `#` comments are ignored. Default: `proxy-api-keys` in the PAIR data directory, consulted only if it exists. On Linux and macOS it must not be readable by group or others. Re-read whenever it changes, so keys can be added, rotated, or revoked without a restart. | +| `NVPAIR_PROXY_API_KEYS` | Comma-separated keys supplied inline, for headless or containerized nodes. Combined with the file. | +| `NVPAIR_PROXY_ALLOWED_CIDRS` | Optional comma-separated CIDR prefixes; a non-loopback caller outside every prefix is refused before its key is examined. | + +A key must be at least 32 printable ASCII characters with no whitespace. Any invalid entry, an unreadable or over-permissive key file, or a malformed CIDR disables the gate (the LAN stays closed) and the reason is logged. Keys are held in memory only as SHA-256 digests and compared in constant time; a refused request is logged with the caller's address and the first eight hex digits of the presented key's digest, never the key. Enabling the gate is logged at warning level. Refusals are `401` `unauthorized` (with `WWW-Authenticate: Bearer realm="nvpair-proxy"`) for a missing or unknown key and `403` `source-not-allowed` for a caller outside the allowlist. An `OPTIONS` preflight is still answered `204` ahead of the gate, since a browser sends no `Authorization` on a preflight. + **Persisted port.** A port chosen at runtime via the `set-port` request (see below) is saved as `lmstudio-proxy-port.json` in the per-user data dir (`%LocalAppData%\Nvidia Corporation\Personal AI Router` on Windows, `~/.config/Nvidia Corporation/Personal AI Router` on Linux) and **restored on startup**, taking precedence over `--port`/the default. One value is exempt: a stored `1235` is discarded and `--port` is used instead, so that port cannot be restored even when it was chosen deliberately via `set-port`. Any other stored port is honoured. The broker uses `--ignore-persisted-port` while reserving the managed `1234` facade. diff --git a/services/ollama-proxy/README.md b/services/ollama-proxy/README.md index 35f959b5..49f52e71 100644 --- a/services/ollama-proxy/README.md +++ b/services/ollama-proxy/README.md @@ -35,7 +35,18 @@ ollama-proxy [flags] The proxy listens on `--port` (default 11435) and forwards incoming HTTP requests to the currently active Ollama node — except the model-list routes `GET /api/tags` and `GET /v1/models`, which are queried across every candidate node concurrently and merged into one de-duplicated inventory. Point your Ollama client at `http://localhost:11435` and the proxy handles routing. When the broker supplies `--alias-address`, the proxy reserves that loopback-only endpoint before reporting ready and serves it through the same routing and workload-lifecycle handler. A `localhost` alias reserves `127.0.0.1` and `::1` atomically so client resolution cannot bypass the router. An occupied alias is non-fatal: its existing owner is untouched and the proxy reports an actionable warning while the primary listener stays available. -**Cluster ingress.** The listener carries two personalities, demultiplexed by each connection's first byte. Plaintext HTTP is accepted only from loopback; a LAN caller is refused. When `--cluster-dir` shows this node is a cluster member, the same listener also terminates cluster mTLS: a peer whose client certificate matches one of this node's pins is forwarded straight to the local engine reported by `node/set-local-backend`, and is never re-routed onward to another node. Membership and pins are re-derived per request, so joining or leaving a cluster needs no restart. +**Cluster ingress.** The listener carries two personalities, demultiplexed by each connection's first byte. Plaintext HTTP is accepted from loopback; a LAN caller is refused unless the operator has enabled authenticated LAN access (next paragraph). When `--cluster-dir` shows this node is a cluster member, the same listener also terminates cluster mTLS: a peer whose client certificate matches one of this node's pins is forwarded straight to the local engine reported by `node/set-local-backend`, and is never re-routed onward to another node. Membership and pins are re-derived per request, so joining or leaving a cluster needs no restart. + +**Authenticated LAN access (opt-in).** With no API key configured, a non-loopback plaintext request is refused with `403` `loopback-only`. An operator enables LAN access by configuring at least one key; the proxy then admits a non-loopback caller that presents a configured key as `Authorization: Bearer ` or `X-Api-Key: `, routes it exactly like a loopback client, and strips the key before forwarding so it never reaches an engine or a peer. Loopback callers are never asked for a key. The gate is `nvpair-shared/ingressauth`, shared with [`lmstudio-proxy`](../lmstudio-proxy/README.md) so both proxies enforce it identically. + +| Variable | Meaning | +|---|---| +| `NVPAIR_PROXY_API_KEYS_FILE` | Key file, one key per line; blank lines and `#` comments are ignored. Default: `proxy-api-keys` in the PAIR data directory, consulted only if it exists. On Linux and macOS it must not be readable by group or others. Re-read whenever it changes, so keys can be added, rotated, or revoked without a restart. | +| `NVPAIR_PROXY_API_KEYS` | Comma-separated keys supplied inline, for headless or containerized nodes. Combined with the file. | +| `NVPAIR_PROXY_ALLOWED_CIDRS` | Optional comma-separated CIDR prefixes; a non-loopback caller outside every prefix is refused before its key is examined. | + +A key must be at least 32 printable ASCII characters with no whitespace. Any invalid entry, an unreadable or over-permissive key file, or a malformed CIDR disables the gate (the LAN stays closed) and the reason is logged. Keys are held in memory only as SHA-256 digests and compared in constant time; a refused request is logged with the caller's address and the first eight hex digits of the presented key's digest, never the key. Enabling the gate is logged at warning level. Refusals are `401` `unauthorized` (with `WWW-Authenticate: Bearer realm="nvpair-proxy"`) for a missing or unknown key and `403` `source-not-allowed` for a caller outside the allowlist. An `OPTIONS` preflight is still answered `204` ahead of the gate, since a browser sends no `Authorization` on a preflight. + **Persisted port.** A port chosen at runtime via the `set-port` request (see below) is saved as `proxy-port.json` in the per-user data dir (`%LocalAppData%\Nvidia Corporation\Personal AI Router` on Windows, `~/.config/Nvidia Corporation/Personal AI Router` on Linux) and **restored on startup**, taking precedence over `--port`/the default — so the proxy comes back up where it was last put. `--ignore-persisted-port` deliberately bypasses that restoration for a broker-coordinated start. Delete the file (or `set-port` back to the default) to revert. From a4ca63083c52e87ec5fd86c87dfe968b16f7e749 Mon Sep 17 00:00:00 2001 From: "Aaron K. Clark (CryptoJones)" Date: Sat, 5 Sep 2026 20:12:47 -0500 Subject: [PATCH 4/4] Harden the API-key gate after independent review Read the key file through the opened handle and re-read it by content hash rather than a stat stamp: a stat-then-open check validated a file it did not necessarily read, and a stamp of size and modification time cannot see a same-length rewrite within the filesystem's timestamp granularity, which is exactly the rotation a compromised key needs. Re-check at most once per second so a node that never opted in does not pay an open() per LAN request. Refuse a key file another user owns, the way sshd treats authorized_keys, and fail closed when ownership cannot be determined. Judge each request from one Authorize call that also reports whether the gate is enabled, so enablement and the key set cannot change between the two questions. Apply the CIDR allowlist before answering a preflight, so a source the operator excluded gets nothing. Accept the key from either the Authorization or X-Api-Key header when both are present, since an SDK may send a placeholder Bearer beside the real key. Carry error="invalid_token" on a rejected key per RFC 6750, restrict keys to the b64token alphabet so they survive any conformant intermediary, and log a key rotation even when the key count is unchanged, as SECURITY.md promises. Documentation: a TLS-terminating or reverse proxy on the same host presents every client as loopback and must do its own authentication; a key reaches inference routed to peers and Ollama's model-management routes; inline environment keys remain in the process environment in clear; Windows relies on the data directory's ACL; any same-user process can enable the gate by creating the file. Add fuzz targets for the parsers and header extraction, a concurrent-rotation test, and the isLoopbackRemote table test to lmstudio-proxy. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01BwxtwuoRxP75PdR6NAmMS3 Signed-off-by: Aaron K. Clark (CryptoJones) --- SECURITY.md | 34 +- docs/getting-started.mdx | 19 +- services/lmstudio-proxy/README.md | 8 +- services/lmstudio-proxy/ingress.go | 36 +- services/lmstudio-proxy/ingress_auth_test.go | 38 ++ services/lmstudio-proxy/ingress_test.go | 18 + services/ollama-proxy/README.md | 8 +- services/ollama-proxy/ingress.go | 36 +- services/ollama-proxy/ingress_auth_test.go | 38 ++ services/shared/ingressauth/ingressauth.go | 331 ++++++++++++------ .../shared/ingressauth/ingressauth_test.go | 260 +++++++++++++- services/shared/ingressauth/owner_unix.go | 31 ++ .../shared/ingressauth/owner_unix_test.go | 59 ++++ services/shared/ingressauth/owner_windows.go | 13 + 14 files changed, 760 insertions(+), 169 deletions(-) create mode 100644 services/shared/ingressauth/owner_unix.go create mode 100644 services/shared/ingressauth/owner_unix_test.go create mode 100644 services/shared/ingressauth/owner_windows.go diff --git a/SECURITY.md b/SECURITY.md index e8548cbf..eb3fff26 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -87,15 +87,35 @@ request is routed like a loopback client, and the key is stripped before the request is forwarded, so it never reaches an engine or a peer. Loopback callers are never asked for a key. -The gate compares keys in constant time, holds them in memory only as digests, -never logs a presented key (a rejection logs a short digest fingerprint), and -fails closed: a key file that other users can read, that contains a malformed -entry, or that cannot be read contributes no keys and the LAN stays closed. +A key therefore reaches everything a loopback client on that node can reach: +inference routed onward to other paired nodes, and every engine route the proxy +forwards, which for Ollama includes model management (`/api/pull`, +`/api/delete`, `/api/create`, `/api/copy`). Treat it as a credential for the +cluster, not for one machine. Enabling the gate restricts the network, not the +host: on a multi-user machine, every local account keeps loopback access to the +proxy regardless of the gate. Any process running as the proxy's own user can +also enable the gate by creating the key file, just as it could set the +environment; the proxy announces that at warning level in its log. + +The loopback exemption also means that anything terminating connections on the +node itself and re-originating them locally — a reverse proxy or TLS terminator +on the same host, or Docker Desktop on macOS forwarding container traffic to the +host — presents every one of its clients to PAIR as a loopback caller that is +never asked for a key. Such a front end must enforce its own authentication, or +run on a different host so that PAIR sees its real address. + +The gate compares keys in constant time, holds file keys in memory only as +digests (a key supplied inline through the environment also remains in the +process environment in clear, so prefer the file), never logs a presented key +(a rejection logs a short digest fingerprint), and fails closed: a key file that other users can read (judged by Unix permission +bits; on Windows the check is skipped and the per-user data directory's ACL is +the protection), that contains a malformed entry, or that cannot be read +contributes no keys and the LAN stays closed. Enabling the gate is logged at warning level at startup and whenever the key set changes. It adds no TLS, rate limiting, or per-key permissions: the plaintext -personality stays plaintext, so use it only on a network you trust or behind a -TLS-terminating proxy you control, and treat a configured key as a credential -that grants everything a local application can do. +personality stays plaintext, so use it only on a network you trust, and pair it +with `NVPAIR_PROXY_ALLOWED_CIDRS` so a leaked key is only usable from the +networks you expect. ### Local Network Is a Trust-Relevant Boundary diff --git a/docs/getting-started.mdx b/docs/getting-started.mdx index c89cf629..46b43777 100644 --- a/docs/getting-started.mdx +++ b/docs/getting-started.mdx @@ -456,8 +456,9 @@ Kubernetes workload, an SDK on a machine you do not administer. For those, a node can accept requests from the network when the caller presents an API key. The default is unchanged. With no key configured, the endpoint stays local. -1. Generate a key. Anything of at least 32 printable ASCII characters with no - whitespace works; a random one is best: +1. Generate a key: at least 32 characters drawn from letters, digits, and + `- . _ ~ + / =`, produced randomly. Nothing limits how fast a caller can + guess, so a memorable passphrase is not a substitute: ```bash openssl rand -hex 32 @@ -473,7 +474,9 @@ The default is unchanged. With no key configured, the endpoint stays local. | macOS | `~/Library/Application Support/Nvidia Corporation/Personal AI Router/proxy-api-keys` | On Linux and macOS the file must be readable by you alone (`chmod 600`). A - file other users can read is ignored, and the node's log says so. + file other users can read is ignored, and the node's log says so. On + Windows the default location is inside your own `%LOCALAPPDATA%`, which + other accounts cannot read by default; PAIR does not check the ACL itself. 3. Point the client at the node's address and the port shown in **Endpoints**, for example `http://gpu-box:11434` or `http://gpu-box:1234/v1`, and send the @@ -488,7 +491,10 @@ The default is unchanged. With no key configured, the endpoint stays local. `X-Api-Key: ` instead. The proxy notices the key file appearing, changing, or disappearing on its own, -so you can add, rotate, or revoke keys without restarting PAIR. Three environment +so you can add, rotate, or revoke keys without restarting PAIR. When you replace +it from a script, write the new file beside the old one and rename it into place; +a rewrite that truncates the file first is briefly seen as an empty key file and +refused until the next check, about a second later. Three environment variables refine the behavior for headless or containerized nodes: `NVPAIR_PROXY_API_KEYS_FILE` names a different key file, `NVPAIR_PROXY_API_KEYS` supplies keys inline (comma-separated), and `NVPAIR_PROXY_ALLOWED_CIDRS` (for @@ -500,7 +506,10 @@ responses cross your network unencrypted, and anyone holding the key can use the node, and through it every node in the cluster, exactly as a local application could. Use it on a network you trust, keep the key private, and prefer the CIDR allowlist. Applications on the node itself keep working over loopback without a -key. The node logs a warning when authenticated access is enabled and logs every +key, and so does anything that terminates connections on the node and forwards +them locally, such as a reverse proxy or TLS terminator on the same machine: PAIR +sees those clients as loopback and never asks them for a key, so such a front +end must do its own authentication. The node logs a warning when authenticated access is enabled and logs every request it refuses, identifying a rejected key only by a short fingerprint. ## Verify It Is Working diff --git a/services/lmstudio-proxy/README.md b/services/lmstudio-proxy/README.md index f3eb5fa4..9ee1d619 100644 --- a/services/lmstudio-proxy/README.md +++ b/services/lmstudio-proxy/README.md @@ -38,15 +38,15 @@ The proxy listens on `--port` (default 1234) and forwards incoming HTTP requests **Cluster ingress.** The listener carries two personalities, demultiplexed by each connection's first byte. Plaintext HTTP is accepted from loopback; a LAN caller is refused unless the operator has enabled authenticated LAN access (next paragraph). When `--cluster-dir` shows this node is a cluster member, the same listener also terminates cluster mTLS: a peer whose client certificate matches one of this node's pins is forwarded straight to the local engine reported by `node/set-local-backend`, and is never re-routed onward to another node. Membership and pins are re-derived per request, so joining or leaving a cluster needs no restart. -**Authenticated LAN access (opt-in).** With no API key configured, a non-loopback plaintext request is refused with `403` `loopback-only`. An operator enables LAN access by configuring at least one key; the proxy then admits a non-loopback caller that presents a configured key as `Authorization: Bearer ` or `X-Api-Key: `, routes it exactly like a loopback client, and strips the key before forwarding so it never reaches an engine or a peer. Loopback callers are never asked for a key. The gate is `nvpair-shared/ingressauth`, shared with [`ollama-proxy`](../ollama-proxy/README.md) so both proxies enforce it identically. +**Authenticated LAN access (opt-in).** With no API key configured, a non-loopback plaintext request is refused with `403` `loopback-only`. An operator enables LAN access by configuring at least one key; the proxy then admits a non-loopback caller that presents a configured key as `Authorization: Bearer ` or `X-Api-Key: ` (either header may carry it; a client that sends a placeholder Bearer token alongside a real `X-Api-Key` is admitted), routes it exactly like a loopback client, and strips the key before forwarding so it never reaches an engine or a peer. Loopback callers are never asked for a key. The gate is `nvpair-shared/ingressauth`, shared with [`ollama-proxy`](../ollama-proxy/README.md) so both proxies enforce it identically. | Variable | Meaning | |---|---| -| `NVPAIR_PROXY_API_KEYS_FILE` | Key file, one key per line; blank lines and `#` comments are ignored. Default: `proxy-api-keys` in the PAIR data directory, consulted only if it exists. On Linux and macOS it must not be readable by group or others. Re-read whenever it changes, so keys can be added, rotated, or revoked without a restart. | +| `NVPAIR_PROXY_API_KEYS_FILE` | Key file, one key per line; blank lines and `#` comments are ignored. Default: `proxy-api-keys` in the PAIR data directory, consulted only if it exists. On Linux and macOS it must be owned by the proxy's user (or root) and not be readable by group or others; on Windows the mode bits are not checked and the per-user `%LOCALAPPDATA%` data directory's ACL is the protection. Re-read whenever it changes, so keys can be added, rotated, or revoked without a restart; replace it by writing a new file and renaming it into place, since a truncating rewrite is briefly seen as an empty file and refused until the next check. | | `NVPAIR_PROXY_API_KEYS` | Comma-separated keys supplied inline, for headless or containerized nodes. Combined with the file. | -| `NVPAIR_PROXY_ALLOWED_CIDRS` | Optional comma-separated CIDR prefixes; a non-loopback caller outside every prefix is refused before its key is examined. | +| `NVPAIR_PROXY_ALLOWED_CIDRS` | Optional comma-separated CIDR prefixes; a non-loopback caller outside every prefix is refused before its key is examined. Addresses are compared in their unmapped form, so list IPv4 ranges as IPv4 (`192.168.1.0/24`, not `::ffff:192.168.1.0/120`). | -A key must be at least 32 printable ASCII characters with no whitespace. Any invalid entry, an unreadable or over-permissive key file, or a malformed CIDR disables the gate (the LAN stays closed) and the reason is logged. Keys are held in memory only as SHA-256 digests and compared in constant time; a refused request is logged with the caller's address and the first eight hex digits of the presented key's digest, never the key. Enabling the gate is logged at warning level. Refusals are `401` `unauthorized` (with `WWW-Authenticate: Bearer realm="nvpair-proxy"`) for a missing or unknown key and `403` `source-not-allowed` for a caller outside the allowlist. An `OPTIONS` preflight is still answered `204` ahead of the gate, since a browser sends no `Authorization` on a preflight. +A key must be at least 32 characters drawn from letters, digits, and `- . _ ~ + / =` (the output of `openssl rand -hex 32`, or any base64 encoder, qualifies), and should be generated randomly: nothing limits how fast a caller can guess, so a memorable passphrase is not a substitute. Any invalid entry, an unreadable or over-permissive key file, or a malformed CIDR disables the gate (the LAN stays closed) and the reason is logged. Keys are held in memory only as SHA-256 digests and compared in constant time; a refused request is logged with the caller's address and the first eight hex digits of the presented key's digest, never the key. Enabling the gate is logged at warning level. Refusals are `401` `unauthorized` (with `WWW-Authenticate: Bearer realm="nvpair-proxy"`) for a missing or unknown key and `403` `source-not-allowed` for a caller outside the allowlist. An `OPTIONS` preflight is still answered `204` ahead of the gate, since a browser sends no `Authorization` on a preflight. **Persisted port.** A port chosen at runtime via the `set-port` request (see below) is saved as `lmstudio-proxy-port.json` in the per-user data dir (`%LocalAppData%\Nvidia Corporation\Personal AI Router` on Windows, `~/.config/Nvidia Corporation/Personal AI Router` on Linux) and **restored on startup**, taking precedence over `--port`/the default. One value is exempt: a stored `1235` is discarded and `--port` is used instead, so that port cannot be restored even when it was chosen deliberately via `set-port`. Any other stored port is honoured. The broker uses `--ignore-persisted-port` while reserving the managed `1234` facade. diff --git a/services/lmstudio-proxy/ingress.go b/services/lmstudio-proxy/ingress.go index d07ad772..37346bf6 100644 --- a/services/lmstudio-proxy/ingress.go +++ b/services/lmstudio-proxy/ingress.go @@ -13,6 +13,7 @@ import ( "strconv" "nvpair-shared/cors" + "nvpair-shared/ingressauth" ) const engineIdentityProbeHeader = "X-NVPAIR-Engine-Identity-Probe" @@ -65,34 +66,49 @@ func (p *Proxy) localBackendTarget() (*url.URL, bool) { // the TLS personality, but plaintext is loopback-only unless authenticated. func (p *Proxy) handlePlain(w http.ResponseWriter, r *http.Request) { if !isLoopbackRemote(r.RemoteAddr) { - // Answer a non-loopback preflight ahead of the gate. It grants no access - // on its own; the request that follows still receives the real 401/403, - // and a browser sends no Authorization on a preflight anyway. A loopback - // preflight continues into handleHTTP so an available engine's exact - // origin and credentials policy can be preserved. + // One Authorize call refreshes the key file and judges the request from + // that single view, so enablement and the key set cannot change between + // "is the gate on?" and "is this key good?". + var d ingressauth.Decision + if p.lanAuth != nil { + d = p.lanAuth.Authorize(r) + } + // A source outside the operator's allowlist gets nothing — not even the + // preflight — so the allowlist means what it says for OPTIONS too. + if d.Enabled && d.Code == ingressauth.CodeSourceNotAllowed { + slog.Warn("rejected non-loopback plaintext request", "remote", r.RemoteAddr, + "method", r.Method, "path", r.URL.Path, "code", d.Code) + writeIngressError(w, d.Status, d.Code, d.Message) + return + } + // Answer a non-loopback preflight ahead of the credential check. It + // grants no access on its own; the request that follows still receives + // the real 401/403, and a browser sends no Authorization on a preflight + // anyway. A loopback preflight continues into handleHTTP so an available + // engine's exact origin and credentials policy can be preserved. if cors.WritePreflight(w, r) { return } - if p.lanAuth == nil || !p.lanAuth.Enabled() { + if !d.Enabled { slog.Warn("rejected non-loopback plaintext request; cluster peers must use mTLS", "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path) writeIngressError(w, http.StatusForbidden, "loopback-only", "plaintext requests are accepted only from loopback; cluster peers must use the mTLS ingress") return } - d := p.lanAuth.Authorize(r) if !d.Allowed { slog.Warn("rejected non-loopback plaintext request", "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path, "code", d.Code, "key_fp", d.KeyFingerprint) - if d.Status == http.StatusUnauthorized { - w.Header().Set("WWW-Authenticate", `Bearer realm="nvpair-proxy"`) + if d.Challenge != "" { + w.Header().Set("WWW-Authenticate", d.Challenge) } writeIngressError(w, d.Status, d.Code, d.Message) return } // The key is the proxy's credential, not the engine's: never forward it. p.lanAuth.StripCredential(r.Header) - slog.Debug("authenticated non-loopback plaintext request", "remote", r.RemoteAddr, + // At Info, not Debug: a production log must show who used a key. + slog.Info("authenticated non-loopback plaintext request", "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path, "key_fp", d.KeyFingerprint) } // Engine-manager marks identity/action requests so the federated model-list diff --git a/services/lmstudio-proxy/ingress_auth_test.go b/services/lmstudio-proxy/ingress_auth_test.go index 36b8f43f..a67e1d71 100644 --- a/services/lmstudio-proxy/ingress_auth_test.go +++ b/services/lmstudio-proxy/ingress_auth_test.go @@ -144,6 +144,10 @@ func TestHandlePlainNonLoopbackWrongKeyIs401(t *testing.T) { if got := ingressCode(t, rec); got != ingressauth.CodeUnauthorized { t.Errorf("code = %q, want %q", got, ingressauth.CodeUnauthorized) } + // RFC 6750 §3.1: a credential that was examined and rejected is told so. + if got := rec.Header().Get("WWW-Authenticate"); !strings.Contains(got, `error="invalid_token"`) { + t.Errorf("WWW-Authenticate = %q, want error=\"invalid_token\" on a rejected key", got) + } if seen.Load() != nil { t.Fatal("a request with a wrong key reached the engine") } @@ -240,3 +244,37 @@ func TestHandlePlainGateWithoutKeysKeepsLoopbackOnly(t *testing.T) { t.Fatal("a LAN request reached the engine with no keys configured") } } + +// TestHandlePlainPreflightOutsideAllowedCIDRIs403: the allowlist applies to a +// preflight too. A source the operator excluded gets no 204 that would let a +// browser proceed to the request that follows. +func TestHandlePlainPreflightOutsideAllowedCIDRIs403(t *testing.T) { + p, seen := lanProxy(t, netip.MustParsePrefix("10.0.0.0/8")) + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodOptions, "/v1/chat/completions", nil) + req.RemoteAddr = lanRemote + p.handlePlain(rec, req) + + if rec.Code != http.StatusForbidden { + t.Fatalf("out-of-allowlist preflight status = %d, want 403", rec.Code) + } + if got := ingressCode(t, rec); got != ingressauth.CodeSourceNotAllowed { + t.Errorf("code = %q, want %q", got, ingressauth.CodeSourceNotAllowed) + } + if seen.Load() != nil { + t.Fatal("a preflight reached the engine") + } +} + +// TestHandlePlainPreflightInsideAllowedCIDRIs204: inside the allowlist the +// preflight is still answered without a credential, as browsers require. +func TestHandlePlainPreflightInsideAllowedCIDRIs204(t *testing.T) { + p, _ := lanProxy(t, netip.MustParsePrefix("192.0.2.0/24")) + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodOptions, "/v1/chat/completions", nil) + req.RemoteAddr = lanRemote + p.handlePlain(rec, req) + if rec.Code != http.StatusNoContent { + t.Fatalf("in-allowlist preflight status = %d, want 204", rec.Code) + } +} diff --git a/services/lmstudio-proxy/ingress_test.go b/services/lmstudio-proxy/ingress_test.go index 2f1e1869..20660631 100644 --- a/services/lmstudio-proxy/ingress_test.go +++ b/services/lmstudio-proxy/ingress_test.go @@ -109,3 +109,21 @@ func TestLocalReverseProxyUsesSharedPlainTransport(t *testing.T) { t.Fatal("ingress reverse proxy did not use the shared plain Transport") } } + +func TestIsLoopbackRemote(t *testing.T) { + for _, c := range []struct { + addr string + want bool + }{ + {"127.0.0.1:5000", true}, + {"[::1]:5000", true}, + {"192.168.1.10:5000", false}, + {"10.0.0.5:80", false}, + {"", false}, + {"garbage", false}, + } { + if got := isLoopbackRemote(c.addr); got != c.want { + t.Errorf("isLoopbackRemote(%q) = %v, want %v", c.addr, got, c.want) + } + } +} diff --git a/services/ollama-proxy/README.md b/services/ollama-proxy/README.md index 49f52e71..234d4578 100644 --- a/services/ollama-proxy/README.md +++ b/services/ollama-proxy/README.md @@ -37,15 +37,15 @@ The proxy listens on `--port` (default 11435) and forwards incoming HTTP request **Cluster ingress.** The listener carries two personalities, demultiplexed by each connection's first byte. Plaintext HTTP is accepted from loopback; a LAN caller is refused unless the operator has enabled authenticated LAN access (next paragraph). When `--cluster-dir` shows this node is a cluster member, the same listener also terminates cluster mTLS: a peer whose client certificate matches one of this node's pins is forwarded straight to the local engine reported by `node/set-local-backend`, and is never re-routed onward to another node. Membership and pins are re-derived per request, so joining or leaving a cluster needs no restart. -**Authenticated LAN access (opt-in).** With no API key configured, a non-loopback plaintext request is refused with `403` `loopback-only`. An operator enables LAN access by configuring at least one key; the proxy then admits a non-loopback caller that presents a configured key as `Authorization: Bearer ` or `X-Api-Key: `, routes it exactly like a loopback client, and strips the key before forwarding so it never reaches an engine or a peer. Loopback callers are never asked for a key. The gate is `nvpair-shared/ingressauth`, shared with [`lmstudio-proxy`](../lmstudio-proxy/README.md) so both proxies enforce it identically. +**Authenticated LAN access (opt-in).** With no API key configured, a non-loopback plaintext request is refused with `403` `loopback-only`. An operator enables LAN access by configuring at least one key; the proxy then admits a non-loopback caller that presents a configured key as `Authorization: Bearer ` or `X-Api-Key: ` (either header may carry it; a client that sends a placeholder Bearer token alongside a real `X-Api-Key` is admitted), routes it exactly like a loopback client, and strips the key before forwarding so it never reaches an engine or a peer. Loopback callers are never asked for a key. The gate is `nvpair-shared/ingressauth`, shared with [`lmstudio-proxy`](../lmstudio-proxy/README.md) so both proxies enforce it identically. | Variable | Meaning | |---|---| -| `NVPAIR_PROXY_API_KEYS_FILE` | Key file, one key per line; blank lines and `#` comments are ignored. Default: `proxy-api-keys` in the PAIR data directory, consulted only if it exists. On Linux and macOS it must not be readable by group or others. Re-read whenever it changes, so keys can be added, rotated, or revoked without a restart. | +| `NVPAIR_PROXY_API_KEYS_FILE` | Key file, one key per line; blank lines and `#` comments are ignored. Default: `proxy-api-keys` in the PAIR data directory, consulted only if it exists. On Linux and macOS it must be owned by the proxy's user (or root) and not be readable by group or others; on Windows the mode bits are not checked and the per-user `%LOCALAPPDATA%` data directory's ACL is the protection. Re-read whenever it changes, so keys can be added, rotated, or revoked without a restart; replace it by writing a new file and renaming it into place, since a truncating rewrite is briefly seen as an empty file and refused until the next check. | | `NVPAIR_PROXY_API_KEYS` | Comma-separated keys supplied inline, for headless or containerized nodes. Combined with the file. | -| `NVPAIR_PROXY_ALLOWED_CIDRS` | Optional comma-separated CIDR prefixes; a non-loopback caller outside every prefix is refused before its key is examined. | +| `NVPAIR_PROXY_ALLOWED_CIDRS` | Optional comma-separated CIDR prefixes; a non-loopback caller outside every prefix is refused before its key is examined. Addresses are compared in their unmapped form, so list IPv4 ranges as IPv4 (`192.168.1.0/24`, not `::ffff:192.168.1.0/120`). | -A key must be at least 32 printable ASCII characters with no whitespace. Any invalid entry, an unreadable or over-permissive key file, or a malformed CIDR disables the gate (the LAN stays closed) and the reason is logged. Keys are held in memory only as SHA-256 digests and compared in constant time; a refused request is logged with the caller's address and the first eight hex digits of the presented key's digest, never the key. Enabling the gate is logged at warning level. Refusals are `401` `unauthorized` (with `WWW-Authenticate: Bearer realm="nvpair-proxy"`) for a missing or unknown key and `403` `source-not-allowed` for a caller outside the allowlist. An `OPTIONS` preflight is still answered `204` ahead of the gate, since a browser sends no `Authorization` on a preflight. +A key must be at least 32 characters drawn from letters, digits, and `- . _ ~ + / =` (the output of `openssl rand -hex 32`, or any base64 encoder, qualifies), and should be generated randomly: nothing limits how fast a caller can guess, so a memorable passphrase is not a substitute. Any invalid entry, an unreadable or over-permissive key file, or a malformed CIDR disables the gate (the LAN stays closed) and the reason is logged. Keys are held in memory only as SHA-256 digests and compared in constant time; a refused request is logged with the caller's address and the first eight hex digits of the presented key's digest, never the key. Enabling the gate is logged at warning level. Refusals are `401` `unauthorized` (with `WWW-Authenticate: Bearer realm="nvpair-proxy"`) for a missing or unknown key and `403` `source-not-allowed` for a caller outside the allowlist. An `OPTIONS` preflight is still answered `204` ahead of the gate, since a browser sends no `Authorization` on a preflight. **Persisted port.** A port chosen at runtime via the `set-port` request (see below) is saved as `proxy-port.json` in the per-user data dir (`%LocalAppData%\Nvidia Corporation\Personal AI Router` on Windows, `~/.config/Nvidia Corporation/Personal AI Router` on Linux) and **restored on startup**, taking precedence over `--port`/the default — so the proxy comes back up where it was last put. `--ignore-persisted-port` deliberately bypasses that restoration for a broker-coordinated start. Delete the file (or `set-port` back to the default) to revert. diff --git a/services/ollama-proxy/ingress.go b/services/ollama-proxy/ingress.go index 1d5e646b..a9d29ddd 100644 --- a/services/ollama-proxy/ingress.go +++ b/services/ollama-proxy/ingress.go @@ -13,6 +13,7 @@ import ( "strconv" "nvpair-shared/cors" + "nvpair-shared/ingressauth" ) const engineIdentityProbeHeader = "X-NVPAIR-Engine-Identity-Probe" @@ -65,34 +66,49 @@ func (p *Proxy) localBackendTarget() (*url.URL, bool) { // the TLS personality, but plaintext is loopback-only unless authenticated. func (p *Proxy) handlePlain(w http.ResponseWriter, r *http.Request) { if !isLoopbackRemote(r.RemoteAddr) { - // Answer a non-loopback preflight ahead of the gate. It grants no access - // on its own; the request that follows still receives the real 401/403, - // and a browser sends no Authorization on a preflight anyway. A loopback - // preflight continues into handleHTTP so an available engine's exact - // origin and credentials policy can be preserved. + // One Authorize call refreshes the key file and judges the request from + // that single view, so enablement and the key set cannot change between + // "is the gate on?" and "is this key good?". + var d ingressauth.Decision + if p.lanAuth != nil { + d = p.lanAuth.Authorize(r) + } + // A source outside the operator's allowlist gets nothing — not even the + // preflight — so the allowlist means what it says for OPTIONS too. + if d.Enabled && d.Code == ingressauth.CodeSourceNotAllowed { + slog.Warn("rejected non-loopback plaintext request", "remote", r.RemoteAddr, + "method", r.Method, "path", r.URL.Path, "code", d.Code) + writeIngressError(w, d.Status, d.Code, d.Message) + return + } + // Answer a non-loopback preflight ahead of the credential check. It + // grants no access on its own; the request that follows still receives + // the real 401/403, and a browser sends no Authorization on a preflight + // anyway. A loopback preflight continues into handleHTTP so an available + // engine's exact origin and credentials policy can be preserved. if cors.WritePreflight(w, r) { return } - if p.lanAuth == nil || !p.lanAuth.Enabled() { + if !d.Enabled { slog.Warn("rejected non-loopback plaintext request; cluster peers must use mTLS", "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path) writeIngressError(w, http.StatusForbidden, "loopback-only", "plaintext requests are accepted only from loopback; cluster peers must use the mTLS ingress") return } - d := p.lanAuth.Authorize(r) if !d.Allowed { slog.Warn("rejected non-loopback plaintext request", "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path, "code", d.Code, "key_fp", d.KeyFingerprint) - if d.Status == http.StatusUnauthorized { - w.Header().Set("WWW-Authenticate", `Bearer realm="nvpair-proxy"`) + if d.Challenge != "" { + w.Header().Set("WWW-Authenticate", d.Challenge) } writeIngressError(w, d.Status, d.Code, d.Message) return } // The key is the proxy's credential, not the engine's: never forward it. p.lanAuth.StripCredential(r.Header) - slog.Debug("authenticated non-loopback plaintext request", "remote", r.RemoteAddr, + // At Info, not Debug: a production log must show who used a key. + slog.Info("authenticated non-loopback plaintext request", "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path, "key_fp", d.KeyFingerprint) } // Engine-manager marks its private identity/action requests so this diff --git a/services/ollama-proxy/ingress_auth_test.go b/services/ollama-proxy/ingress_auth_test.go index eb48b76c..39700bb9 100644 --- a/services/ollama-proxy/ingress_auth_test.go +++ b/services/ollama-proxy/ingress_auth_test.go @@ -144,6 +144,10 @@ func TestHandlePlainNonLoopbackWrongKeyIs401(t *testing.T) { if got := ingressCode(t, rec); got != ingressauth.CodeUnauthorized { t.Errorf("code = %q, want %q", got, ingressauth.CodeUnauthorized) } + // RFC 6750 §3.1: a credential that was examined and rejected is told so. + if got := rec.Header().Get("WWW-Authenticate"); !strings.Contains(got, `error="invalid_token"`) { + t.Errorf("WWW-Authenticate = %q, want error=\"invalid_token\" on a rejected key", got) + } if seen.Load() != nil { t.Fatal("a request with a wrong key reached the engine") } @@ -240,3 +244,37 @@ func TestHandlePlainGateWithoutKeysKeepsLoopbackOnly(t *testing.T) { t.Fatal("a LAN request reached the engine with no keys configured") } } + +// TestHandlePlainPreflightOutsideAllowedCIDRIs403: the allowlist applies to a +// preflight too. A source the operator excluded gets no 204 that would let a +// browser proceed to the request that follows. +func TestHandlePlainPreflightOutsideAllowedCIDRIs403(t *testing.T) { + p, seen := lanProxy(t, netip.MustParsePrefix("10.0.0.0/8")) + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodOptions, "/v1/chat/completions", nil) + req.RemoteAddr = lanRemote + p.handlePlain(rec, req) + + if rec.Code != http.StatusForbidden { + t.Fatalf("out-of-allowlist preflight status = %d, want 403", rec.Code) + } + if got := ingressCode(t, rec); got != ingressauth.CodeSourceNotAllowed { + t.Errorf("code = %q, want %q", got, ingressauth.CodeSourceNotAllowed) + } + if seen.Load() != nil { + t.Fatal("a preflight reached the engine") + } +} + +// TestHandlePlainPreflightInsideAllowedCIDRIs204: inside the allowlist the +// preflight is still answered without a credential, as browsers require. +func TestHandlePlainPreflightInsideAllowedCIDRIs204(t *testing.T) { + p, _ := lanProxy(t, netip.MustParsePrefix("192.0.2.0/24")) + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodOptions, "/v1/chat/completions", nil) + req.RemoteAddr = lanRemote + p.handlePlain(rec, req) + if rec.Code != http.StatusNoContent { + t.Fatalf("in-allowlist preflight status = %d, want 204", rec.Code) + } +} diff --git a/services/shared/ingressauth/ingressauth.go b/services/shared/ingressauth/ingressauth.go index f6ad20a6..76a97f07 100644 --- a/services/shared/ingressauth/ingressauth.go +++ b/services/shared/ingressauth/ingressauth.go @@ -27,6 +27,7 @@ package ingressauth import ( "bufio" + "bytes" "crypto/sha256" "crypto/subtle" "encoding/hex" @@ -42,7 +43,6 @@ import ( "strings" "sync" "time" - "unicode" "nvpair-shared/appdir" ) @@ -62,10 +62,29 @@ const ( DefaultKeyFileName = "proxy-api-keys" // MinKeyLength is the shortest key the gate accepts. A 32-character key - // drawn from hex already carries 128 bits, which puts online guessing out - // of reach without a lockout mechanism. + // drawn at random from the allowed alphabet carries well over 128 bits, + // which puts online guessing out of reach without a lockout mechanism. A + // 32-character passphrase does not; the documentation says to generate + // keys randomly. MinKeyLength = 32 + // MaxKeyLength bounds a key, configured or presented. A credential a + // client sends is hashed before it is compared, so without a ceiling a + // caller could make the proxy digest a megabyte of header per request; + // anything longer than this is not a key and is not examined. + MaxKeyLength = 512 + + // maxKeyFileBytes bounds how much of a key file is read. A key file holds + // a handful of short lines; anything larger is not a key file. + maxKeyFileBytes = 64 << 10 + + // defaultRecheckEvery is how long a FromEnv gate trusts its last look at + // the key file before opening it again. Without a floor, a node that never + // opted in would still pay an open() for every unauthenticated LAN request, + // which is a remotely triggered cost it did not have before. One second + // keeps rotation and revocation effectively immediate. + defaultRecheckEvery = time.Second + // CodeUnauthorized is the ingress error code for a missing or wrong key. CodeUnauthorized = "unauthorized" // CodeSourceNotAllowed is the ingress error code for a caller outside the @@ -77,33 +96,40 @@ const ( bearerScheme = "bearer" noCredential = "none" + // challengeMissing and challengeInvalid are the WWW-Authenticate values for + // a 401, per RFC 6750 §3: a request with no credential gets the bare + // challenge; one whose credential was examined and rejected also carries + // error="invalid_token". + challengeMissing = `Bearer realm="nvpair-proxy"` + challengeInvalid = `Bearer realm="nvpair-proxy", error="invalid_token"` + unauthorizedMessage = "a valid API key is required for non-loopback requests; " + "send it as Authorization: Bearer or X-Api-Key: " ) -// Decision is the gate's verdict on one request. When Allowed is false, Status, -// Code, and Message are what the proxy should answer with, in the same shape as -// its other ingress rejections. KeyFingerprint identifies the presented key for -// the log without revealing it; it is "none" when no credential was sent. +// Decision is the gate's verdict on one request. Enabled reports whether any +// key is configured at the moment of the call; when it is false the proxy +// applies its loopback-only refusal and the other fields are unset. When +// Enabled is true and Allowed is false, Status, Code, and Message are what the +// proxy should answer with, in the same shape as its other ingress rejections, +// and Challenge, when non-empty, is the WWW-Authenticate value to send. +// KeyFingerprint identifies the presented key for the log without revealing it; +// it is "none" when no credential was examined. type Decision struct { + Enabled bool Allowed bool Status int Code string Message string + Challenge string KeyFingerprint string } -type digest = [sha256.Size]byte +// digest is a SHA-256 of a key. A distinct type, so a raw byte array cannot be +// mistaken for one. +type digest [sha256.Size]byte -// fileStamp is the part of a key file's metadata that decides whether it must -// be re-read. Mode is included because fixing permissions with chmod changes -// neither size nor modification time, yet must take effect. -type fileStamp struct { - size int64 - modTime time.Time - mode fs.FileMode - exists bool -} +func digestOf(key string) digest { return digest(sha256.Sum256([]byte(key))) } // Gate holds the configured credentials and allowlist. Its zero value is a // disabled gate; construct one with FromEnv or New. @@ -114,25 +140,36 @@ type Gate struct { inline []digest // broken records an unrecoverable configuration error in the environment // (a malformed inline key or CIDR). The gate then stays disabled for the - // life of the process, regardless of the key file. + // life of the process, regardless of the key file. The proxy keeps serving + // loopback clients; taking it down for an optional setting would punish + // the desktop application for an operator's typo. broken bool - // filePath, when non-empty, is re-checked on every Enabled call so keys can - // be rotated without restarting the proxy. explicitFile records that the - // operator named the path, so its absence is worth reporting. + // filePath, when non-empty, is re-read (at most once per recheckEvery) on + // Authorize so keys can be rotated without restarting the proxy. The + // cached keys are reused while the file's content hash is unchanged. + // explicitFile records that the operator named the path, so its absence + // is worth reporting. fileErr is the last error logged for the file, so a + // persisting problem is reported once rather than per request and a new + // problem is reported when it appears. filePath string explicitFile bool fileKeys []digest - fileStamp fileStamp - fileChecked bool + fileHash digest + fileLoaded bool + fileErr string + recheckEvery time.Duration + lastCheck time.Time cidrs []netip.Prefix // announced* remember the last state written to the log, so a change is - // reported once rather than on every request. + // reported once rather than on every request. The file hash is part of the + // state so a rotation that keeps the key count is still reported. announcedOnce bool announcedEnabled bool announcedKeys int + announcedHash digest } // New builds a gate from literal keys and prefixes, for tests and callers that @@ -145,7 +182,7 @@ func New(keys []string, cidrs []netip.Prefix) *Gate { if err := validateKey(k); err != nil { panic("ingressauth.New: " + err.Error()) } - g.inline = append(g.inline, sha256.Sum256([]byte(k))) + g.inline = append(g.inline, digestOf(k)) } g.mu.Lock() g.announceLocked() @@ -157,7 +194,7 @@ func New(keys []string, cidrs []netip.Prefix) *Gate { // configuration error is logged and yields a gate that stays disabled, which // leaves the proxy in its loopback-only default. func FromEnv() *Gate { - g := &Gate{} + g := &Gate{recheckEvery: defaultRecheckEvery} if raw := os.Getenv(EnvKeys); strings.TrimSpace(raw) != "" { for i, k := range strings.Split(raw, ",") { @@ -171,7 +208,7 @@ func FromEnv() *Gate { g.broken = true break } - g.inline = append(g.inline, sha256.Sum256([]byte(k))) + g.inline = append(g.inline, digestOf(k)) } } @@ -209,8 +246,9 @@ func FromEnv() *Gate { } // Enabled reports whether at least one API key is configured, re-reading the -// key file first if it changed. The proxy consults this per non-loopback -// request, so adding, rotating, or removing keys needs no restart. +// key file first if it is due. The proxy does not call this per request — +// Authorize reports the same thing in its Decision from a single refresh — but +// it is the natural question for startup logging and tests. func (g *Gate) Enabled() bool { g.mu.Lock() defer g.mu.Unlock() @@ -223,44 +261,53 @@ func (g *Gate) enabledLocked() bool { return !g.broken && len(g.inline)+len(g.fileKeys) > 0 } -// Authorize decides whether a non-loopback plaintext request may proceed. The -// allowlist is checked before the credential, so a caller outside it learns -// nothing about whether its key is valid. Authorize does not write to the -// response; the proxy does, in its own error format. +// Authorize decides whether a non-loopback plaintext request may proceed. It +// refreshes the key file once and answers from that single view, so the +// enabled/disabled state and the key set a request is judged against cannot +// change between two calls. The allowlist is checked before the credential, +// so a caller outside it learns nothing about whether its key is valid — and +// the proxy applies that source decision even to a preflight, which needs no +// credential. Authorize does not write to the response; the proxy does, in +// its own error format. func (g *Gate) Authorize(r *http.Request) Decision { g.mu.Lock() g.refreshLocked() + g.announceLocked() + enabled := g.enabledLocked() cidrs := g.cidrs digests := make([]digest, 0, len(g.inline)+len(g.fileKeys)) digests = append(digests, g.inline...) digests = append(digests, g.fileKeys...) - enabled := g.enabledLocked() g.mu.Unlock() if !enabled { - // The proxy only asks an enabled gate; answer conservatively anyway. - return Decision{Status: http.StatusForbidden, Code: CodeSourceNotAllowed, - Message: "authenticated LAN ingress is not enabled", KeyFingerprint: noCredential} + return Decision{} } if len(cidrs) > 0 { ip, ok := remoteAddr(r) if !ok || !anyPrefixContains(cidrs, ip) { - return Decision{Status: http.StatusForbidden, Code: CodeSourceNotAllowed, + return Decision{Enabled: true, Status: http.StatusForbidden, Code: CodeSourceNotAllowed, Message: "the caller's address is outside " + EnvAllowedCIDRs, KeyFingerprint: noCredential} } } - cred, ok := credentialFrom(r) - if !ok { - return Decision{Status: http.StatusUnauthorized, Code: CodeUnauthorized, - Message: unauthorizedMessage, KeyFingerprint: noCredential} + creds := credentialsFrom(r) + if len(creds) == 0 { + return Decision{Enabled: true, Status: http.StatusUnauthorized, Code: CodeUnauthorized, + Message: unauthorizedMessage, Challenge: challengeMissing, KeyFingerprint: noCredential} } - if !matchesAny(digests, sha256.Sum256([]byte(cred))) { - return Decision{Status: http.StatusUnauthorized, Code: CodeUnauthorized, - Message: unauthorizedMessage, KeyFingerprint: Fingerprint(cred)} + // Either presented credential may match. An SDK that always sends a + // placeholder Bearer token alongside the real X-Api-Key must not be locked + // out by header precedence; both headers are the caller's to set, so + // checking both costs nothing in security. + for _, cred := range creds { + if matchesAny(digests, digestOf(cred)) { + return Decision{Enabled: true, Allowed: true, KeyFingerprint: fingerprint(cred)} + } } - return Decision{Allowed: true, Status: http.StatusOK, KeyFingerprint: Fingerprint(cred)} + return Decision{Enabled: true, Status: http.StatusUnauthorized, Code: CodeUnauthorized, + Message: unauthorizedMessage, Challenge: challengeInvalid, KeyFingerprint: fingerprint(creds[0])} } // StripCredential removes the presented key from a request the gate admitted, @@ -270,17 +317,22 @@ func (g *Gate) StripCredential(h http.Header) { h.Del(headerAPIKey) } -// Fingerprint returns the first eight hex characters of a key's SHA-256 digest: +// fingerprint returns the first eight hex characters of a key's SHA-256 digest: // enough for an operator to tell repeated rejections of one misconfigured -// client apart from a scan, without the log ever holding the key. -func Fingerprint(key string) string { +// client apart from a scan, without the log ever holding the key. It is +// deliberately unsalted so an operator can compute it from the key they meant +// to configure and confirm which client is misconfigured; the price is that +// the log holds 32 bits of a truncated hash of a client's key, which is one +// more reason keys must be random rather than memorable. +func fingerprint(key string) string { sum := sha256.Sum256([]byte(key)) return hex.EncodeToString(sum[:4]) } -// matchesAny compares the presented digest against every configured digest in -// constant time and without an early exit, so neither the key length nor the -// position of a match is observable through timing. +// matchesAny compares the presented digest against every configured digest +// with a constant-time comparison and no early exit, so neither the key +// length nor the position of a match is observable through timing. (The +// number of configured keys is not a secret.) func matchesAny(configured []digest, presented digest) bool { match := 0 for i := range configured { @@ -289,22 +341,25 @@ func matchesAny(configured []digest, presented digest) bool { return match == 1 } -// credentialFrom extracts the client's key: a Bearer token first, then the -// X-Api-Key header. A query parameter is deliberately not accepted, because -// URLs end up in access logs and browser histories. -func credentialFrom(r *http.Request) (string, bool) { +// credentialsFrom extracts the client's presented keys: a Bearer token and an +// X-Api-Key header, in that order, whichever are present and no longer than +// MaxKeyLength (an over-long value cannot be a key and is not hashed). A query +// parameter is deliberately not accepted, because URLs end up in access logs +// and browser histories. +func credentialsFrom(r *http.Request) []string { + var creds []string if auth := strings.TrimSpace(r.Header.Get(headerAuthorization)); auth != "" { scheme, token, found := strings.Cut(auth, " ") if found && strings.EqualFold(scheme, bearerScheme) { - if token = strings.TrimSpace(token); token != "" { - return token, true + if token = strings.TrimSpace(token); token != "" && len(token) <= MaxKeyLength { + creds = append(creds, token) } } } - if key := strings.TrimSpace(r.Header.Get(headerAPIKey)); key != "" { - return key, true + if key := strings.TrimSpace(r.Header.Get(headerAPIKey)); key != "" && len(key) <= MaxKeyLength { + creds = append(creds, key) } - return "", false + return creds } // remoteAddr parses the transport-level peer address. Forwarding headers are @@ -327,56 +382,79 @@ func anyPrefixContains(prefixes []netip.Prefix, ip netip.Addr) bool { return false } -// refreshLocked re-reads the key file when its metadata changed since the last -// look. Caller holds g.mu. +// refreshLocked re-reads the key file, at most once per recheckEvery, and +// swaps the cached keys when its content changed. The file is read rather +// than stat-compared: a key file is a few short lines, and a stamp of size +// and modification time cannot see a same-length rewrite within the +// filesystem's timestamp granularity — exactly the rotation a compromised key +// needs. Caller holds g.mu. func (g *Gate) refreshLocked() { if g.filePath == "" || g.broken { return } - info, err := os.Stat(g.filePath) - var stamp fileStamp + now := time.Now() + if !g.lastCheck.IsZero() && now.Sub(g.lastCheck) < g.recheckEvery { + return + } + g.lastCheck = now + + content, err := readKeyFile(g.filePath) switch { case err == nil: - stamp = fileStamp{size: info.Size(), modTime: info.ModTime(), mode: info.Mode(), exists: true} + g.fileErr = "" + hash := digest(sha256.Sum256(content)) + if g.fileLoaded && hash == g.fileHash { + return + } + keys, perr := parseKeys(bytes.NewReader(content)) + if perr != nil { + g.dropFileKeysLocked() + g.logFileErrorLocked("key file ignored; no file keys are in effect", perr) + return + } + g.fileKeys, g.fileHash, g.fileLoaded = keys, hash, true case errors.Is(err, fs.ErrNotExist): - stamp = fileStamp{} - default: - // A stat failure other than absence (a parent directory's permissions, - // an I/O error) counts as absence for this request and is re-examined - // on the next; report it when it is news. - if !g.fileChecked || g.fileStamp.exists { - slog.Error("authenticated LAN ingress: cannot stat key file; no file keys are in effect", - "path", g.filePath, "err", err) + g.dropFileKeysLocked() + if g.explicitFile { + g.logFileErrorLocked("key file does not exist; no file keys are in effect", err) } - stamp = fileStamp{} + default: + g.dropFileKeysLocked() + g.logFileErrorLocked("key file ignored; no file keys are in effect", err) } - if g.fileChecked && stamp == g.fileStamp { +} + +func (g *Gate) dropFileKeysLocked() { + g.fileKeys, g.fileHash, g.fileLoaded = nil, digest{}, false +} + +// logFileErrorLocked reports a key-file problem once per distinct error, so a +// persisting misconfiguration does not write a line per request while a new +// one is still reported the moment it appears. +func (g *Gate) logFileErrorLocked(msg string, err error) { + if err.Error() == g.fileErr { return } - g.fileChecked = true - g.fileStamp = stamp - g.fileKeys = nil + g.fileErr = err.Error() + slog.Error("authenticated LAN ingress: "+msg, "path", g.filePath, "err", err) +} - if !stamp.exists { - if g.explicitFile { - slog.Error("authenticated LAN ingress: key file does not exist; no file keys are in effect", - "env", EnvKeysFile, "path", g.filePath) - } - return +// readKeyFile opens the key file and checks the opened handle — not a separate +// stat — before reading, so the permissions, type, and owner it validates +// belong to the file it reads. On Unix-like systems the file must belong to +// the proxy's user (or root) and must not be readable or writable by group or +// others; on Windows the mode bits carry no such meaning and the check is +// skipped, leaving protection to the data directory's ACL. +func readKeyFile(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err } - keys, err := loadKeyFile(g.filePath, info) + defer f.Close() + info, err := f.Stat() if err != nil { - slog.Error("authenticated LAN ingress: key file ignored; no file keys are in effect", - "path", g.filePath, "err", err) - return + return nil, err } - g.fileKeys = keys -} - -// loadKeyFile reads and validates a key file. On Unix-like systems the file must -// not be readable or writable by group or others; on Windows the mode bits -// carry no such meaning and the check is skipped. -func loadKeyFile(path string, info fs.FileInfo) ([]digest, error) { if !info.Mode().IsRegular() { return nil, errors.New("not a regular file") } @@ -385,12 +463,17 @@ func loadKeyFile(path string, info fs.FileInfo) ([]digest, error) { return nil, fmt.Errorf("permissions %04o allow other users to read it; chmod 600", perm) } } - f, err := os.Open(path) + if err := ownedByProcessUser(info); err != nil { + return nil, err + } + content, err := io.ReadAll(io.LimitReader(f, maxKeyFileBytes+1)) if err != nil { return nil, err } - defer f.Close() - return parseKeys(f) + if len(content) > maxKeyFileBytes { + return nil, fmt.Errorf("larger than %d bytes; not a key file", maxKeyFileBytes) + } + return content, nil } // parseKeys reads one key per line. Blank lines and lines starting with '#' are @@ -410,7 +493,7 @@ func parseKeys(r io.Reader) ([]digest, error) { if err := validateKey(entry); err != nil { return nil, fmt.Errorf("line %d: %w", line, err) } - keys = append(keys, sha256.Sum256([]byte(entry))) + keys = append(keys, digestOf(entry)) } if err := sc.Err(); err != nil { return nil, err @@ -422,40 +505,58 @@ func parseKeys(r io.Reader) ([]digest, error) { } // validateKey enforces the shape a key must have to be usable at all: long -// enough to resist guessing, and printable ASCII with no whitespace so it -// survives an HTTP header unchanged. +// enough to resist guessing, and drawn from the RFC 6750 b64token alphabet +// (letters, digits, and - . _ ~ + / =) so it survives an Authorization header +// unchanged through any conformant intermediary. Hex and base64 output both +// qualify. func validateKey(key string) error { if len(key) < MinKeyLength { return fmt.Errorf("key is %d characters; at least %d are required", len(key), MinKeyLength) } - for _, c := range key { - if c > unicode.MaxASCII || c <= ' ' || c == 0x7f { - return errors.New("key must be printable ASCII with no whitespace") + if len(key) > MaxKeyLength { + return fmt.Errorf("key is %d characters; at most %d are allowed", len(key), MaxKeyLength) + } + for i := 0; i < len(key); i++ { + if !isTokenByte(key[i]) { + return errors.New("key must use only letters, digits, and - . _ ~ + / =") } } return nil } -// announceLocked logs a change in the gate's state — enabled with N keys, or -// back to disabled — once per change. Enabling is logged at Warn: it widens the -// proxy's exposure and an operator reading the log should see it plainly. -// Caller holds g.mu. +func isTokenByte(c byte) bool { + switch { + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9': + return true + } + return strings.IndexByte("-._~+/=", c) >= 0 +} + +// announceLocked logs a change in the gate's state — enabled with N keys, a +// rotation of the key file, or back to disabled — once per change. Enabling +// is logged at Warn: it widens the proxy's exposure and an operator reading +// the log should see it plainly. Caller holds g.mu. func (g *Gate) announceLocked() { enabled := g.enabledLocked() n := len(g.inline) + len(g.fileKeys) - if g.announcedOnce && enabled == g.announcedEnabled && n == g.announcedKeys { + if g.announcedOnce && enabled == g.announcedEnabled && n == g.announcedKeys && g.fileHash == g.announcedHash { return } - g.announcedOnce, g.announcedEnabled, g.announcedKeys = true, enabled, n - if enabled { + rotated := g.announcedOnce && enabled && g.announcedEnabled && g.fileHash != g.announcedHash + g.announcedOnce, g.announcedEnabled, g.announcedKeys, g.announcedHash = true, enabled, n, g.fileHash + switch { + case rotated: + slog.Warn("authenticated LAN ingress: key file changed; the configured key set was replaced", + "keys", n, "key_file", g.filePath) + case enabled: cidrs := make([]string, 0, len(g.cidrs)) for _, p := range g.cidrs { cidrs = append(cidrs, p.String()) } slog.Warn("authenticated LAN ingress ENABLED: a non-loopback plaintext caller presenting a configured API key is routed", "keys", n, "key_file", g.filePath, "allowed_cidrs", cidrs) - return + default: + slog.Info("authenticated LAN ingress disabled; plaintext requests are accepted from loopback only", + "key_file", g.filePath) } - slog.Info("authenticated LAN ingress disabled; plaintext requests are accepted from loopback only", - "key_file", g.filePath) } diff --git a/services/shared/ingressauth/ingressauth_test.go b/services/shared/ingressauth/ingressauth_test.go index c7e2a70d..2d0b6e86 100644 --- a/services/shared/ingressauth/ingressauth_test.go +++ b/services/shared/ingressauth/ingressauth_test.go @@ -4,6 +4,9 @@ package ingressauth import ( + "bytes" + "fmt" + "log/slog" "net/http" "net/http/httptest" "net/netip" @@ -11,6 +14,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "testing" "time" ) @@ -44,6 +48,12 @@ func TestValidateKey(t *testing.T) { {"embedded tab", "0123456789abcdef\t123456789abcdef0", false}, {"non-ascii", "0123456789abcdef0123456789abcdé", false}, {"control char", "0123456789abcdef0123456789abcde\x01", false}, + {"double quote", strings.Repeat("a", MinKeyLength-1) + `"`, false}, + {"backslash", strings.Repeat("a", MinKeyLength-1) + `\`, false}, + {"base64 alphabet", "abcd+/ABCD0123456789abcdef012345==", true}, + {"exactly maximum length", strings.Repeat("k", MaxKeyLength), true}, + {"one over maximum", strings.Repeat("k", MaxKeyLength+1), false}, + {"urlsafe alphabet", "abcd-_ABCD0123456789abcdef012345~.", true}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -98,22 +108,28 @@ func TestAuthorizeCredentials(t *testing.T) { status int fp string }{ - {"bearer first key", []string{"Authorization", "Bearer " + keyA}, true, http.StatusOK, Fingerprint(keyA)}, - {"bearer second key", []string{"Authorization", "Bearer " + keyB}, true, http.StatusOK, Fingerprint(keyB)}, - {"lowercase scheme", []string{"Authorization", "bearer " + keyA}, true, http.StatusOK, Fingerprint(keyA)}, - {"x-api-key", []string{"X-Api-Key", keyA}, true, http.StatusOK, Fingerprint(keyA)}, - {"x-api-key lowercase header name", []string{"x-api-key", keyB}, true, http.StatusOK, Fingerprint(keyB)}, + {"bearer first key", []string{"Authorization", "Bearer " + keyA}, true, http.StatusOK, fingerprint(keyA)}, + {"bearer second key", []string{"Authorization", "Bearer " + keyB}, true, http.StatusOK, fingerprint(keyB)}, + {"lowercase scheme", []string{"Authorization", "bearer " + keyA}, true, http.StatusOK, fingerprint(keyA)}, + {"x-api-key", []string{"X-Api-Key", keyA}, true, http.StatusOK, fingerprint(keyA)}, + {"x-api-key lowercase header name", []string{"x-api-key", keyB}, true, http.StatusOK, fingerprint(keyB)}, {"no credential", nil, false, http.StatusUnauthorized, noCredential}, - {"wrong key", []string{"Authorization", "Bearer " + keyC}, false, http.StatusUnauthorized, Fingerprint(keyC)}, + {"wrong key", []string{"Authorization", "Bearer " + keyC}, false, http.StatusUnauthorized, fingerprint(keyC)}, {"wrong scheme", []string{"Authorization", "Basic " + keyA}, false, http.StatusUnauthorized, noCredential}, {"bearer with no token", []string{"Authorization", "Bearer "}, false, http.StatusUnauthorized, noCredential}, - {"key as prefix only", []string{"Authorization", "Bearer " + keyA + "x"}, false, http.StatusUnauthorized, Fingerprint(keyA + "x")}, - {"wrong bearer but right x-api-key", []string{"Authorization", "Bearer " + keyC, "X-Api-Key", keyA}, false, http.StatusUnauthorized, Fingerprint(keyC)}, + {"key as prefix only", []string{"Authorization", "Bearer " + keyA + "x"}, false, http.StatusUnauthorized, fingerprint(keyA + "x")}, + // An SDK placeholder Bearer beside a real X-Api-Key must not lock the client out. + {"placeholder bearer but right x-api-key", []string{"Authorization", "Bearer " + keyC, "X-Api-Key", keyA}, true, http.StatusOK, fingerprint(keyA)}, + {"right bearer but stale x-api-key", []string{"Authorization", "Bearer " + keyA, "X-Api-Key", keyC}, true, http.StatusOK, fingerprint(keyA)}, + {"both wrong", []string{"Authorization", "Bearer " + keyC, "X-Api-Key", keyC + "x"}, false, http.StatusUnauthorized, fingerprint(keyC)}, + // An over-long value is not a key: it is not hashed, so it counts as no credential. + {"over-long bearer", []string{"Authorization", "Bearer " + strings.Repeat("a", MaxKeyLength+1)}, false, http.StatusUnauthorized, noCredential}, + {"over-long x-api-key beside a valid bearer", []string{"Authorization", "Bearer " + keyA, "X-Api-Key", strings.Repeat("a", 1<<20)}, true, http.StatusOK, fingerprint(keyA)}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { d := g.Authorize(request("192.0.2.9:40000", tc.hdr...)) - if d.Allowed != tc.allow || d.Status != tc.status { + if d.Allowed != tc.allow || (!tc.allow && d.Status != tc.status) { t.Fatalf("decision = %+v, want allowed=%v status=%d", d, tc.allow, tc.status) } if d.KeyFingerprint != tc.fp { @@ -125,8 +141,22 @@ func TestAuthorizeCredentials(t *testing.T) { if strings.Contains(d.Message, keyA) || strings.Contains(d.Message, keyC) { t.Errorf("message echoes a key: %q", d.Message) } + if !d.Enabled { + t.Error("decision from an enabled gate reports Enabled=false") + } }) } + // RFC 6750 §3: a bare challenge when nothing was presented, error="invalid_token" + // when a credential was examined and rejected, no challenge on success. + if d := g.Authorize(request("192.0.2.9:1")); d.Challenge != challengeMissing { + t.Errorf("no-credential challenge = %q, want %q", d.Challenge, challengeMissing) + } + if d := g.Authorize(request("192.0.2.9:1", "X-Api-Key", keyC)); d.Challenge != challengeInvalid { + t.Errorf("wrong-key challenge = %q, want %q", d.Challenge, challengeInvalid) + } + if d := g.Authorize(request("192.0.2.9:1", "X-Api-Key", keyA)); d.Challenge != "" { + t.Errorf("challenge on success = %q, want none", d.Challenge) + } } func TestAuthorizeCIDRAllowlist(t *testing.T) { @@ -190,27 +220,56 @@ func TestDisabledGateNeverAllows(t *testing.T) { if zero.Enabled() { t.Fatal("zero Gate reports enabled") } - if d := zero.Authorize(request("192.0.2.9:1", "Authorization", "Bearer "+keyA)); d.Allowed { - t.Fatalf("zero Gate allowed a request: %+v", d) + if d := zero.Authorize(request("192.0.2.9:1", "Authorization", "Bearer "+keyA)); d.Allowed || d.Enabled { + t.Fatalf("zero Gate decision = %+v, want neither enabled nor allowed", d) } empty := New(nil, nil) if empty.Enabled() { t.Fatal("New(nil, nil) reports enabled") } + if d := empty.Authorize(request("192.0.2.9:1", "Authorization", "Bearer "+keyA)); d.Allowed || d.Enabled { + t.Fatalf("keyless Gate decision = %+v, want neither enabled nor allowed", d) + } +} + +// TestKeyFileSameSizeSameTimeRewriteIsNoticed: rotating a key for another of +// the same length, within the filesystem's timestamp granularity, must take +// effect — a stamp of size and mtime cannot see it, so the gate hashes content. +func TestKeyFileSameSizeSameTimeRewriteIsNoticed(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "keys") + g := &Gate{filePath: path, explicitFile: true} + when := time.Now().Add(-time.Hour).Truncate(time.Second) + keyA2 := strings.ToUpper(keyA) // same length, different bytes + if len(keyA2) != len(keyA) || keyA2 == keyA { + t.Fatal("test keys must differ only in content") + } + + writeKeyFile(t, path, keyA+"\n", 0o600, when) + if d := g.Authorize(request("192.0.2.9:1", "X-Api-Key", keyA)); !d.Allowed { + t.Fatalf("initial key rejected: %+v", d) + } + writeKeyFile(t, path, keyA2+"\n", 0o600, when) // identical size, mtime, and mode + if d := g.Authorize(request("192.0.2.9:1", "X-Api-Key", keyA)); d.Allowed { + t.Fatal("rotated-out key still accepted after a same-size, same-time rewrite") + } + if d := g.Authorize(request("192.0.2.9:1", "X-Api-Key", keyA2)); !d.Allowed { + t.Fatalf("rotated-in key rejected: %+v", d) + } } func TestFingerprintIsShortStableHexAndNotTheKey(t *testing.T) { - fp := Fingerprint(keyA) + fp := fingerprint(keyA) if len(fp) != 8 { t.Fatalf("fingerprint %q has length %d, want 8", fp, len(fp)) } if strings.ToLower(fp) != fp || strings.Trim(fp, "0123456789abcdef") != "" { t.Fatalf("fingerprint %q is not lowercase hex", fp) } - if fp != Fingerprint(keyA) { + if fp != fingerprint(keyA) { t.Fatal("fingerprint is not stable") } - if fp == Fingerprint(keyB) { + if fp == fingerprint(keyB) { t.Fatal("distinct keys share a fingerprint") } if strings.Contains(keyA, fp) { @@ -454,3 +513,176 @@ func TestFromEnvExplicitMissingFileIsDisabled(t *testing.T) { t.Fatal("enabled with a missing explicit key file") } } + +// The parsers face operator files and attacker-controlled headers; none of them +// may panic on arbitrary bytes, and anything validateKey accepts must be a key +// the gate can actually match over the wire. +func FuzzValidateKey(f *testing.F) { + for _, seed := range []string{"", keyA, keyB, keyC, "short", "0123456789abcdef 123456789abcdef0", "é", "\x00\xff"} { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, key string) { + if err := validateKey(key); err == nil { + if len(key) < MinKeyLength { + t.Fatalf("accepted %d-character key", len(key)) + } + for i := 0; i < len(key); i++ { + if !isTokenByte(key[i]) { + t.Fatalf("accepted key with byte %q", key[i]) + } + } + } + }) +} + +func FuzzParseKeys(f *testing.F) { + for _, seed := range []string{"", "# c\n", keyA + "\n", keyA + "\r\n" + keyB + "\n", "\x00\n\xff", strings.Repeat("a", 4096)} { + f.Add([]byte(seed)) + } + f.Fuzz(func(t *testing.T, data []byte) { + keys, err := parseKeys(bytes.NewReader(data)) + if err == nil && len(keys) == 0 { + t.Fatal("no error and no keys") + } + }) +} + +func FuzzCredentialsFrom(f *testing.F) { + for _, seed := range [][2]string{{"Bearer " + keyA, ""}, {"bearer", ""}, {"Basic x", keyA}, {"", "\x00"}, {"Bearer ", " "}} { + f.Add(seed[0], seed[1]) + } + f.Fuzz(func(t *testing.T, auth, apiKey string) { + r := request("192.0.2.9:1") + r.Header.Set("Authorization", auth) + r.Header.Set("X-Api-Key", apiKey) + for _, c := range credentialsFrom(r) { + if c == "" || c != strings.TrimSpace(c) || len(c) > MaxKeyLength { + t.Fatalf("extracted credential %q is empty, untrimmed, or over-long", c) + } + } + }) +} + +// captureLog routes slog to a buffer for the test's duration. +func captureLog(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + t.Cleanup(func() { slog.SetDefault(prev) }) + return &buf +} + +// TestRotationIsLogged: SECURITY.md promises a log line whenever the key set +// changes, and a rotation that keeps the key count is the case an operator +// most needs to see. +func TestRotationIsLogged(t *testing.T) { + path := filepath.Join(t.TempDir(), "keys") + g := &Gate{filePath: path, explicitFile: true} + when := time.Now().Add(-time.Hour) + buf := captureLog(t) + + writeKeyFile(t, path, keyA+"\n", 0o600, when) + g.Enabled() + if !strings.Contains(buf.String(), "ENABLED") { + t.Fatalf("enabling not logged:\n%s", buf.String()) + } + buf.Reset() + writeKeyFile(t, path, keyB+"\n", 0o600, when.Add(time.Second)) + g.Enabled() + if !strings.Contains(buf.String(), "key file changed") { + t.Fatalf("rotation with an unchanged key count not logged:\n%s", buf.String()) + } + if strings.Contains(buf.String(), keyA) || strings.Contains(buf.String(), keyB) { + t.Fatal("a key reached the log") + } + buf.Reset() + g.Enabled() + if buf.Len() != 0 { + t.Fatalf("unchanged state logged again:\n%s", buf.String()) + } +} + +// TestRecheckFloorBoundsFileReads: a FromEnv gate re-reads the key file at most +// once per recheckEvery, so a node that never opted in does not pay an open() +// for every unauthenticated LAN request. +func TestRecheckFloorBoundsFileReads(t *testing.T) { + path := filepath.Join(t.TempDir(), "keys") + g := &Gate{filePath: path, explicitFile: true, recheckEvery: time.Hour} + when := time.Now().Add(-time.Hour) + + writeKeyFile(t, path, keyA+"\n", 0o600, when) + if !g.Enabled() { + t.Fatal("first look did not load the file") + } + writeKeyFile(t, path, keyB+"\n", 0o600, when.Add(time.Second)) + if d := g.Authorize(request("192.0.2.9:1", "X-Api-Key", keyA)); !d.Allowed { + t.Fatal("the file was re-read inside the recheck window") + } + g.lastCheck = time.Time{} // window elapsed + if d := g.Authorize(request("192.0.2.9:1", "X-Api-Key", keyB)); !d.Allowed { + t.Fatalf("rotated key not picked up after the window: %+v", d) + } + clearEnv(t) + if got := FromEnv().recheckEvery; got != defaultRecheckEvery { + t.Fatalf("FromEnv recheckEvery = %v, want %v", got, defaultRecheckEvery) + } + _ = fmt.Sprint +} + +// TestConcurrentAuthorizeDuringRotation drives Authorize from many goroutines +// while the key file is rewritten underneath; run under -race. Every decision +// must be for exactly one of the two keys that were ever valid — never neither, +// never both — and a never-configured key must never pass. +func TestConcurrentAuthorizeDuringRotation(t *testing.T) { + path := filepath.Join(t.TempDir(), "keys") + g := &Gate{filePath: path, explicitFile: true} + when := time.Now().Add(-time.Hour) + writeKeyFile(t, path, keyA+"\n", 0o600, when) + + var wg sync.WaitGroup + stop := make(chan struct{}) + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + // Both answers must come from one view of the file, so a + // rotation between the two calls must not be confused with a + // state where neither or both keys are valid: judge a single + // request carrying both keys, which Authorize checks together. + both := g.Authorize(request("192.0.2.9:1", "Authorization", "Bearer "+keyA, "X-Api-Key", keyB)) + if !both.Allowed { + t.Error("neither of the two ever-valid keys was accepted") + return + } + if g.Authorize(request("192.0.2.9:1", "X-Api-Key", keyC)).Allowed { + t.Error("a never-configured key was accepted") + return + } + } + }() + } + // Rotate the way an operator should: write the new file beside the old one + // and rename it into place, so no reader ever sees a truncated file. (A + // truncating rewrite would be seen as an empty key file for one recheck — + // correctly fail-closed, but not what this test is about.) + for i := 1; i <= 20; i++ { + k := keyA + if i%2 == 1 { + k = keyB + } + tmp := path + ".tmp" + writeKeyFile(t, tmp, k+"\n", 0o600, when.Add(time.Duration(i)*time.Second)) + if err := os.Rename(tmp, path); err != nil { + t.Fatal(err) + } + } + close(stop) + wg.Wait() +} diff --git a/services/shared/ingressauth/owner_unix.go b/services/shared/ingressauth/owner_unix.go new file mode 100644 index 00000000..c3d50a14 --- /dev/null +++ b/services/shared/ingressauth/owner_unix.go @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build !windows + +package ingressauth + +import ( + "errors" + "fmt" + "io/fs" + "os" + "syscall" +) + +// ownedByProcessUser refuses a key file that belongs to another account, the +// way sshd treats authorized_keys: a private mode is not enough if someone else +// owns the file and can change its contents or mode at will. root may own the +// file, since an administrator may provision it for a service user. A +// FileInfo without Unix ownership data is refused too: on a Unix-like system +// that is not a file the gate can vouch for. +func ownedByProcessUser(info fs.FileInfo) error { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return errors.New("cannot determine the key file's owner") + } + if uid := int(st.Uid); uid != os.Geteuid() && uid != 0 { + return fmt.Errorf("owned by uid %d, not by the proxy's user (uid %d)", uid, os.Geteuid()) + } + return nil +} diff --git a/services/shared/ingressauth/owner_unix_test.go b/services/shared/ingressauth/owner_unix_test.go new file mode 100644 index 00000000..200a5ee2 --- /dev/null +++ b/services/shared/ingressauth/owner_unix_test.go @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build !windows + +package ingressauth + +import ( + "io/fs" + "os" + "path/filepath" + "syscall" + "testing" +) + +// fakeInfo is a fs.FileInfo whose Sys() the test controls, so ownership cases +// that would otherwise need root (a file owned by someone else) can be +// exercised. +type fakeInfo struct { + fs.FileInfo + sys any +} + +func (f fakeInfo) Sys() any { return f.sys } + +func TestOwnedByProcessUser(t *testing.T) { + path := filepath.Join(t.TempDir(), "keys") + if err := os.WriteFile(path, []byte(keyA+"\n"), 0o600); err != nil { + t.Fatal(err) + } + real, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if err := ownedByProcessUser(real); err != nil { + t.Fatalf("a file this process just created is refused: %v", err) + } + + me := uint32(os.Geteuid()) + cases := []struct { + name string + sys any + ok bool + }{ + {"owned by the process user", &syscall.Stat_t{Uid: me}, true}, + {"owned by root", &syscall.Stat_t{Uid: 0}, true}, + {"owned by another user", &syscall.Stat_t{Uid: me + 1}, false}, + {"no ownership data", nil, false}, + {"foreign Sys type", struct{}{}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := ownedByProcessUser(fakeInfo{FileInfo: real, sys: tc.sys}) + if (err == nil) != tc.ok { + t.Fatalf("ownedByProcessUser err = %v, want ok=%v", err, tc.ok) + } + }) + } +} diff --git a/services/shared/ingressauth/owner_windows.go b/services/shared/ingressauth/owner_windows.go new file mode 100644 index 00000000..5ec1e520 --- /dev/null +++ b/services/shared/ingressauth/owner_windows.go @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build windows + +package ingressauth + +import "io/fs" + +// ownedByProcessUser is a no-op on Windows, where ownership and access are +// expressed through ACLs rather than a uid; the per-user %LOCALAPPDATA% data +// directory that holds the default key file is the protection there. +func ownedByProcessUser(fs.FileInfo) error { return nil }