diff --git a/docs/architecture.mdx b/docs/architecture.mdx index f7e7ffed..eb11f976 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -522,6 +522,7 @@ A default installation listens on these ports: | --- | --- | | `11434` | Ollama-compatible proxy (Ollama itself moves to `11435`+) | | `1234` | OpenAI-compatible proxy (LM Studio moves to `1235`+) | +| `8080` | llama.cpp (`llama-server` in router mode), loopback only, when PAIR installed it | | `14318` | Node hardware and model inventory | | `14319` | Service-error synchronization between nodes | | `14320` | Workload propagation between nodes | diff --git a/docs/engine-lifecycle.mdx b/docs/engine-lifecycle.mdx index 81bf01d9..746392a7 100644 --- a/docs/engine-lifecycle.mdx +++ b/docs/engine-lifecycle.mdx @@ -6,7 +6,7 @@ SPDX-License-Identifier: Apache-2.0 # Managing Engines in NVIDIA Personal AI Router An **engine** is the local inference runtime Personal AI Router (PAIR) uses to -run models. Today that means Ollama or LM Studio on a given machine. PAIR can +run models. Today that means Ollama, LM Studio, or llama.cpp on a given machine. PAIR can install and run those engines for you, or work with a copy you already have. This page explains what you can expect when you install, start, stop, update, or remove an engine. @@ -132,6 +132,33 @@ controlled: Refer to [Engines and Ports](architecture.mdx#engines-and-ports) in the architecture guide. +## llama.cpp + +PAIR can install and own a `llama-server` from the pinned +[ggml-org/llama.cpp](https://github.com/ggml-org/llama.cpp/releases) release +named in its manifest, verified against the release's published SHA-256. It +runs the server in **router mode**: no model is named on the command line, and +the server lists and loads models on demand from the `models` directory inside +its install. Put a GGUF file there, or a sub-directory holding a multi-shard or +`mmproj` pair, and it appears under **Add model** as an available model. + +What is different from Ollama and LM Studio: + +- **No pull.** llama.cpp has no model registry. PAIR does not download GGUFs for + it; copy files into the models directory (or use the llama.cpp Hugging Face + cache) and use the engine's model list. This is deliberate: a download step + would need a model source PAIR does not have a maintainer-aligned policy for. +- **One resident model by default.** The manifest starts the router with + `--models-max 1`, so loading a second model ejects the first. Raise it in the + manifest only on machines with the memory for it. +- **Builds.** The pinned archives are the Vulkan builds on Windows x64 and Linux + x64 (one archive, no separate runtime download) and the native builds on + macOS and arm64. A CUDA build needs a second runtime archive the manifest + format does not express yet. +- **Adopt vs install.** An already-running `llama-server` on the engine's port + is adopted exactly like the other engines; PAIR does not move or stop a + process it did not start. + ## Managing Engines on Other Machines After systems are paired, you can view another machine's engines from diff --git a/services/nvpair-engine-manager/manifests/llamacpp.json b/services/nvpair-engine-manager/manifests/llamacpp.json new file mode 100644 index 00000000..3967fd36 --- /dev/null +++ b/services/nvpair-engine-manager/manifests/llamacpp.json @@ -0,0 +1,280 @@ +{ + "engine": "llamacpp", + "display_name": "llama.cpp", + "manifest_version": 1, + "install": { + "mode": "user" + }, + "runtime": { + "mode": "process", + "args": [ + "--host", + "{host}", + "--port", + "{port}", + "--models-dir", + "{install_dir}/models", + "--models-max", + "1" + ], + "port": 8080, + "bind": "127.0.0.1", + "ready": { + "http": "http://127.0.0.1:{port}/health", + "status": 200, + "timeout_s": 60 + }, + "stop": { + "signal": "term", + "grace_s": 10 + }, + "health": { + "http": "http://127.0.0.1:{port}/health", + "status": 200, + "interval_s": 10 + } + }, + "platforms": { + "windows/amd64": { + "detect": [ + "{install_dir}\\llama-server.exe" + ], + "install": { + "fetch": { + "url": "https://github.com/ggml-org/llama.cpp/releases/download/b10819/llama-b10819-bin-win-vulkan-x64.zip", + "sha256": "4c5ff97b5440024906fc90f67809d84b92d9b77847c7d1a800701a36499e565e" + }, + "run": [ + "tar", + "-xf", + "{download}", + "-C", + "{install_dir}" + ], + "mode": "user" + }, + "uninstall": { + "run": [ + "cmd", + "/c", + "rmdir", + "/s", + "/q", + "{install_dir}" + ] + }, + "runtime": { + "bin": "{install_dir}\\llama-server.exe" + } + }, + "windows/arm64": { + "detect": [ + "{install_dir}\\llama-server.exe" + ], + "install": { + "fetch": { + "url": "https://github.com/ggml-org/llama.cpp/releases/download/b10819/llama-b10819-bin-win-cpu-arm64.zip", + "sha256": "5802d55f633b68bf6dbe574d75f9f47387761fe3b6ddef4193ea9ea423642afb" + }, + "run": [ + "tar", + "-xf", + "{download}", + "-C", + "{install_dir}" + ], + "mode": "user" + }, + "uninstall": { + "run": [ + "cmd", + "/c", + "rmdir", + "/s", + "/q", + "{install_dir}" + ] + }, + "runtime": { + "bin": "{install_dir}\\llama-server.exe" + } + }, + "darwin/arm64": { + "detect": [ + "{install_dir}/llama-server" + ], + "install": { + "fetch": { + "url": "https://github.com/ggml-org/llama.cpp/releases/download/b10819/llama-b10819-bin-macos-arm64.tar.gz", + "sha256": "8933e736495eadfef0731ae32054acfaa75699bf4a6ccba77cd8475db085ec66" + }, + "run": [ + "tar", + "-xzf", + "{download}", + "-C", + "{install_dir}" + ], + "mode": "user" + }, + "uninstall": { + "run": [ + "rm", + "-rf", + "{install_dir}" + ] + }, + "runtime": { + "bin": "{install_dir}/llama-server" + } + }, + "darwin/amd64": { + "detect": [ + "{install_dir}/llama-server" + ], + "install": { + "fetch": { + "url": "https://github.com/ggml-org/llama.cpp/releases/download/b10819/llama-b10819-bin-macos-x64.tar.gz", + "sha256": "04dd13ec03120685bd6e1931e8f1562d2c981ca076a3e63cc44e9a199b37816a" + }, + "run": [ + "tar", + "-xzf", + "{download}", + "-C", + "{install_dir}" + ], + "mode": "user" + }, + "uninstall": { + "run": [ + "rm", + "-rf", + "{install_dir}" + ] + }, + "runtime": { + "bin": "{install_dir}/llama-server" + } + }, + "linux/amd64": { + "detect": [ + "{install_dir}/llama-server" + ], + "install": { + "fetch": { + "url": "https://github.com/ggml-org/llama.cpp/releases/download/b10819/llama-b10819-bin-ubuntu-vulkan-x64.tar.gz", + "sha256": "2175737ab85506e7639fc7f8c84b5247fd607cc6e7030825c6dcadd4279f62e3" + }, + "run": [ + "tar", + "-xzf", + "{download}", + "-C", + "{install_dir}" + ], + "mode": "user" + }, + "uninstall": { + "run": [ + "rm", + "-rf", + "{install_dir}" + ] + }, + "runtime": { + "bin": "{install_dir}/llama-server" + } + }, + "linux/arm64": { + "detect": [ + "{install_dir}/llama-server" + ], + "install": { + "fetch": { + "url": "https://github.com/ggml-org/llama.cpp/releases/download/b10819/llama-b10819-bin-ubuntu-arm64.tar.gz", + "sha256": "6f6f7e1e9b371d4840860a79ccbf4ad6f7da9da76349ce73079e3289b01033ec" + }, + "run": [ + "tar", + "-xzf", + "{download}", + "-C", + "{install_dir}" + ], + "mode": "user" + }, + "uninstall": { + "run": [ + "rm", + "-rf", + "{install_dir}" + ] + }, + "runtime": { + "bin": "{install_dir}/llama-server" + } + } + }, + "actions": { + "list_models": { + "description": "List every GGUF the router can serve: files placed under {install_dir}/models (single files, or a sub-directory for multi-shard / mmproj pairs) plus the llama.cpp HF cache. Router-mode GET /models.", + "http": { + "method": "GET", + "path": "/models" + }, + "result": { + "array": "data", + "field": "id" + } + }, + "loaded_models": { + "description": "Models currently resident in a child llama-server instance (router-mode GET /models, rows whose status.value is \"loaded\").", + "http": { + "method": "GET", + "path": "/models" + }, + "result": { + "array": "data", + "field": "id", + "match": { + "field": "status.value", + "in": [ + "loaded" + ] + } + } + }, + "load_model": { + "description": "Load a model into memory (params: {\"model\": \"\"}). The router spawns a child llama-server for it; --models-max bounds how many stay resident.", + "http": { + "method": "POST", + "path": "/models/load", + "body_schema": { + "model": "string" + } + } + }, + "unload_model": { + "description": "Unload a model from memory (params: {\"model\": \"\"}). The GGUF stays on disk.", + "http": { + "method": "POST", + "path": "/models/unload", + "body_schema": { + "model": "string" + } + } + }, + "chat": { + "description": "OpenAI-compatible chat completion (params: {\"model\": \"\", \"messages\": [...]}). In router mode the model field selects \u2014 and, unless --no-models-autoload is set, loads \u2014 the instance that serves the request.", + "http": { + "method": "POST", + "path": "/v1/chat/completions", + "body_schema": { + "model": "string", + "messages": "array" + } + } + } + } +} diff --git a/services/nvpair-engine-manager/models.go b/services/nvpair-engine-manager/models.go index de89064e..27e09450 100644 --- a/services/nvpair-engine-manager/models.go +++ b/services/nvpair-engine-manager/models.go @@ -4,6 +4,7 @@ package main import ( + "strings" "bytes" "context" "encoding/json" @@ -201,7 +202,7 @@ func extractStringsResult(raw json.RawMessage, spec *ActionResult) ([]string, bo if spec.Match != nil && !matchRow(el, spec.Match) { continue } - fv, ok := el[spec.Field] + fv, ok := lookupField(el, spec.Field) if !ok { continue } @@ -223,7 +224,7 @@ func extractStringsResult(raw json.RawMessage, spec *ActionResult) ([]string, bo // wrong-typed field fails the match, so a row we cannot classify is excluded // rather than counted as loaded. func matchRow(el map[string]json.RawMessage, m *ResultMatch) bool { - fv, ok := el[m.Field] + fv, ok := lookupField(el, m.Field) if !ok { return false } @@ -245,3 +246,36 @@ func matchRow(el map[string]json.RawMessage, m *ResultMatch) bool { } return false } + +// lookupField resolves a field name against a decoded row. A plain name is a +// top-level key; a dotted name ("status.value") descends through nested +// objects one segment at a time. Only the dotted form descends, so a +// top-level key that itself contains a dot is still found by its literal +// name first. Added for engines whose inventory reports state as a nested +// object — llama.cpp's router mode answers GET /models with +// `"status": {"value": "loaded"}` per row. +func lookupField(el map[string]json.RawMessage, field string) (json.RawMessage, bool) { + if v, ok := el[field]; ok { + return v, true + } + if !strings.Contains(field, ".") { + return nil, false + } + cur := el + parts := strings.Split(field, ".") + for i, part := range parts { + v, ok := cur[part] + if !ok { + return nil, false + } + if i == len(parts)-1 { + return v, true + } + var next map[string]json.RawMessage + if err := json.Unmarshal(v, &next); err != nil { + return nil, false + } + cur = next + } + return nil, false +} diff --git a/services/nvpair-engine-manager/models_test.go b/services/nvpair-engine-manager/models_test.go index a9794f80..3ca4ffeb 100644 --- a/services/nvpair-engine-manager/models_test.go +++ b/services/nvpair-engine-manager/models_test.go @@ -382,3 +382,47 @@ func TestSameStringSet(t *testing.T) { }) } } + +// TestLookupFieldNested covers the dotted-path form llama.cpp's router mode +// needs: GET /models reports residency as a nested object, and a plain +// top-level lookup can neither filter on it nor, with a plain key, find it. +func TestLookupFieldNested(t *testing.T) { + row := map[string]json.RawMessage{ + "id": json.RawMessage(`"Qwen3-8B-Q5_K_M.gguf"`), + "status": json.RawMessage(`{"value":"loaded","args":["llama-server","-c","4096"]}`), + "a.b": json.RawMessage(`"literal-dotted-key"`), + } + if v, ok := lookupField(row, "status.value"); !ok || string(v) != `"loaded"` { + t.Fatalf("status.value: ok=%v v=%s", ok, v) + } + if v, ok := lookupField(row, "a.b"); !ok || string(v) != `"literal-dotted-key"` { + t.Fatalf("literal dotted key must win over descent: ok=%v v=%s", ok, v) + } + if _, ok := lookupField(row, "status.missing"); ok { + t.Fatal("missing nested leaf must not resolve") + } + if _, ok := lookupField(row, "id.value"); ok { + t.Fatal("descending through a string must fail, not panic") + } +} + +// TestExtractStringsNestedMatch is the llama.cpp loaded_models contract end +// to end: only rows whose status.value is "loaded" are counted, and a row +// with no status (a model the router has never touched) is excluded rather +// than counted as resident. +func TestExtractStringsNestedMatch(t *testing.T) { + raw := json.RawMessage(`{"data":[ + {"id":"a.gguf","status":{"value":"loaded"}}, + {"id":"b.gguf","status":{"value":"unloaded"}}, + {"id":"c.gguf","status":{"value":"loading"}}, + {"id":"d.gguf"} + ]}`) + spec := &ActionResult{Array: "data", Field: "id", Match: &ResultMatch{Field: "status.value", In: []string{"loaded"}}} + got, ok := extractStringsResult(raw, spec) + if !ok { + t.Fatal("expected an authoritative result") + } + if len(got) != 1 || got[0] != "a.gguf" { + t.Fatalf("loaded set: %v", got) + } +} diff --git a/services/nvpair-engine-manager/registry_test.go b/services/nvpair-engine-manager/registry_test.go index 2ae2c5c6..1cda5b8e 100644 --- a/services/nvpair-engine-manager/registry_test.go +++ b/services/nvpair-engine-manager/registry_test.go @@ -355,6 +355,34 @@ func TestBundledManifestsMerge(t *testing.T) { t.Error("ollama linux/amd64 missing") } + lc, ok := reg.Get("llamacpp") + if !ok { + t.Fatal("llamacpp not loaded") + } + for _, pf := range [][2]string{{"windows", "amd64"}, {"darwin", "arm64"}, {"linux", "amd64"}} { + p, ok := lc.PlatformFor(pf[0], pf[1]) + if !ok { + t.Errorf("llamacpp %s/%s missing", pf[0], pf[1]) + continue + } + // PAIR-installed llama.cpp: a pinned, checksummed release archive, + // a router-mode runtime (no model on the command line, a models + // directory under the install), and an identity probe on /health. + if p.Install == nil || p.Install.Fetch == nil || p.Install.Fetch.SHA256 == "" { + t.Errorf("llamacpp %s/%s: install must fetch a checksummed archive", pf[0], pf[1]) + } + if p.Runtime.Port != 8080 || p.Runtime.Bin == "" || p.Runtime.Ready == nil || p.Runtime.Ready.HTTP == "" { + t.Errorf("llamacpp %s/%s runtime: port=%d bin=%q ready=%v", pf[0], pf[1], p.Runtime.Port, p.Runtime.Bin, p.Runtime.Ready) + } + joined := strings.Join(p.Runtime.Args, " ") + if !strings.Contains(joined, "--models-dir") || strings.Contains(joined, " -m ") || strings.Contains(joined, "--model ") { + t.Errorf("llamacpp %s/%s must run in router mode (--models-dir, no -m): %q", pf[0], pf[1], joined) + } + } + if a, ok := lc.Actions["loaded_models"]; !ok || a.Result == nil || a.Result.Match == nil || a.Result.Match.Field != "status.value" { + t.Errorf("llamacpp loaded_models must filter on the router's nested status.value") + } + lm, ok := reg.Get("lmstudio") if !ok { t.Fatal("lmstudio not loaded") diff --git a/services/versions.json b/services/versions.json index 29d8c230..eac4adc4 100644 --- a/services/versions.json +++ b/services/versions.json @@ -12,7 +12,7 @@ "nvpair-errors": "0.7.4", "nvpair-node-settings": "1.0.4", "nvpair-ui-broker": "0.40.2", - "nvpair-engine-manager": "0.17.4", + "nvpair-engine-manager": "0.18.0", "nvpair-cluster-manager": "1.1.4", "nvpair-job-scheduler": "0.4.1", "nvpair-tui": "0.7.2"