From 8b147a6df9ffd0f0e0369ebe7af24abce443e324 Mon Sep 17 00:00:00 2001 From: Alec Fong Date: Mon, 24 Aug 2026 15:09:05 -0700 Subject: [PATCH] Discover Pi Webster models dynamically --- .github/workflows/bootstrap.yml | 2 + README.md | 41 ++-- pi-setup.sh | 328 +++++++++++++++++++++++++++----- test.sh | 145 ++++++++++++++ 4 files changed, 451 insertions(+), 65 deletions(-) diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index 2eefba3..d88131d 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -104,6 +104,7 @@ jobs: env: WEBSTER_API_KEY: ci-test-key PI_SKIP_ENDPOINT_CHECK: "1" + PI_MODEL: ci-test-model run: cat pi-setup.sh | bash - name: Verify pi health check passes standalone @@ -115,6 +116,7 @@ jobs: env: WEBSTER_API_KEY: ci-test-key PI_SKIP_ENDPOINT_CHECK: "1" + PI_MODEL: ci-test-model run: | ./pi-setup.sh | tee pi-rerun.log grep -q "tokps-session extension unchanged" pi-rerun.log diff --git a/README.md b/README.md index 54f7318..1d5d4e3 100644 --- a/README.md +++ b/README.md @@ -389,8 +389,10 @@ secret and an existing Claude login. `pi-setup.sh` is the sibling of `omp-setup.sh` for the [pi](https://www.npmjs.com/package/@earendil-works/pi-coding-agent) coding -agent: same `webster` endpoint, same **GLM 5.2** default, plus a footer status -line showing generation speed, active model, and session id. +agent: same `webster` endpoint, every model accessible to the supplied key, +plus a footer status line showing generation speed, active model, and session +id. The installer preserves an accessible existing default, otherwise prefers +`glm-5.2` when available, then falls back to the first discovered model. Hand this to someone and they are set up in one command: @@ -415,8 +417,8 @@ What it writes: | Where | What | |---|---| -| `~/.pi/agent/models.json` | `webster` provider: base URL, API key, `glm-5.2` with real limits (mode `0600`) | -| `~/.pi/agent/settings.json` | `defaultProvider: webster`, `defaultModel: glm-5.2` | +| `~/.pi/agent/models.json` | `webster` provider: base URL, API key, and the key's discovered models with per-model limits (mode `0600`) | +| `~/.pi/agent/settings.json` | `defaultProvider: webster`, plus an accessible discovered default model | | `~/.pi/agent/extensions/tokps-session.ts` | Footer line: `73.4 tok/s • webster/glm-5.2 • 019feddc-…` | **pi needs Node.js ≥ 22.19**, and the script installs or upgrades it @@ -430,12 +432,16 @@ that surfaces as a clear message here instead. A pi already on `PATH` that cannot start is reinstalled rather than reported as present, so re-running after a Node upgrade actually repairs it. -Unlike omp, pi has no endpoint discovery, so the model is declared with the -backend's real limits rather than the proxy's metadata — the proxy reports a -null context window, while vLLM rejects `max_tokens` above -`max_model_len=320000`. Override with `PI_CONTEXT_WINDOW` / `PI_MAX_TOKENS`, -or target a different endpoint entirely with `PI_PROVIDER` / `PI_BASE_URL` / -`PI_MODEL`. +Pi itself has no endpoint discovery, so the installer queries the authenticated +`/v1/models` endpoint and writes every returned model into Pi's configuration. +Context windows come from each model's endpoint metadata, with the previous +320,000-token default used only when metadata is absent. Advertised output is +capped at 32,768 tokens by default because Pi reserves output inside the +context window; smaller advertised limits are honored. `PI_CONTEXT_WINDOW` and +`PI_MAX_TOKENS` override those values for every discovered model, while +`PI_MODEL` explicitly chooses the default and fails if that model is not +accessible. `PI_PROVIDER` and `PI_BASE_URL` can target another compatible +endpoint. The footer rate is **decode speed** — output tokens over the time from first streamed content to end of generation — so a turn that runs a slow bash command @@ -451,13 +457,14 @@ and provisioning. Verification runs on every install and under `--check`, per [STYLE_GUIDE.md](STYLE_GUIDE.md): the provider and default model as they landed -on disk, that `pi` starts with the extension loaded, and that the endpoint -actually accepts the key. A rejected key (HTTP 401/403) is a failure, not a -warning — otherwise a typo'd paste only surfaces on the first prompt. An -unreachable endpoint just warns, so offline machines still get configured -(`PI_SKIP_ENDPOINT_CHECK=1` skips it outright). Failure paths — no key, -unreadable `--key-file`, unparseable `models.json`, a rejected key, and -credential file permissions — are covered in `test.sh`. +on disk, that `pi` starts with the extension loaded, and that the installed +model list still exactly matches the key's current endpoint access. A rejected +key (HTTP 401/403) or an accepted key with no models is a failure. An offline +re-run preserves the previously installed list; a fresh offline install needs +an explicit `PI_MODEL`. `PI_SKIP_ENDPOINT_CHECK=1` uses the same fallback. +Failure paths — no key, unreadable `--key-file`, unparseable `models.json`, a +rejected or empty-access key, and credential file permissions — are covered in +`test.sh`. Re-running is safe: other providers survive a merge, an unparseable config is left untouched, and the extension is only rewritten when its content differs. diff --git a/pi-setup.sh b/pi-setup.sh index 1335b02..45a5d23 100755 --- a/pi-setup.sh +++ b/pi-setup.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # # pi-setup.sh — install the pi coding agent and point it at the Brev-hosted -# "webster" model endpoint, with glm-5.2 as the default model and a footer -# status line (tok/s, active model, session id). +# "webster" model endpoint, discovering the models accessible to the supplied +# key and adding a footer status line (tok/s, active model, session id). # -# The sibling of omp-setup.sh: same endpoint and model, different agent. +# The sibling of omp-setup.sh: same endpoint, different agent. # # Installs the npm package, writes ~/.pi/agent/models.json (provider + models), # sets the default provider/model in ~/.pi/agent/settings.json, and drops the @@ -96,12 +96,21 @@ DEFAULT_BASE_URL="https://webster-models-extnode-3gdrajbr0hiykknxzitck9yaiwo.app PROVIDER="${PI_PROVIDER:-webster}" BASE_URL="${PI_BASE_URL:-$DEFAULT_BASE_URL}" -MODEL="${PI_MODEL:-glm-5.2}" -# The endpoint's real limits: vLLM rejects max_tokens beyond max_model_len, -# and pi reserves output tokens against the context window, so these must -# match the backend rather than the proxy's advertised metadata. -CONTEXT_WINDOW="${PI_CONTEXT_WINDOW:-320000}" -MAX_TOKENS="${PI_MAX_TOKENS:-32768}" +PREFERRED_MODEL="glm-5.2" +MODEL="${PI_MODEL:-}" +MODEL_EXPLICIT=0 +[ -n "$MODEL" ] && MODEL_EXPLICIT=1 + +# Discovery normally supplies each model's context limit. These values are +# only a fallback for an explicit manual model or a cap on advertised output; +# pi reserves maxTokens inside contextWindow, so an endpoint that advertises +# both as the same huge number would otherwise leave no useful prompt space. +DEFAULT_CONTEXT_WINDOW=320000 +DEFAULT_MAX_TOKENS=32768 +CONTEXT_WINDOW_OVERRIDE="${PI_CONTEXT_WINDOW:-}" +MAX_TOKENS_OVERRIDE="${PI_MAX_TOKENS:-}" +DISCOVERED_MODELS='[]' +DISCOVERY_STATUS="" # pi reads its config from PI_CODING_AGENT_DIR, defaulting to ~/.pi/agent. AGENT_DIR="${PI_CODING_AGENT_DIR:-$HOME/.pi/agent}" @@ -122,7 +131,7 @@ VERIFY_ONLY=0 # `curl ... | bash` there is no script on disk to read. usage() { cat <<'USAGE' -pi-setup.sh — install pi and point it at the Brev-hosted webster endpoint. +pi-setup.sh — install pi and discover models from the Brev-hosted webster endpoint. ./pi-setup.sh configure, merging into any existing setup ./pi-setup.sh --exclusive make webster the ONLY configured provider @@ -364,10 +373,214 @@ jq_edit() { return 1 } -# The provider entry plus the model definitions pi needs. pi has no endpoint -# discovery, so every model has to be declared with its real limits. +# normalize_models_response RESPONSE_FILE — convert the OpenAI-compatible +# /models response into the model definitions pi requires. The API supplies a +# context limit per model. Output is capped unless PI_MAX_TOKENS explicitly +# overrides it because pi reserves maxTokens inside contextWindow. +normalize_models_response() { + local response_file="$1" + jq -ce \ + --arg context_override "$CONTEXT_WINDOW_OVERRIDE" \ + --arg max_override "$MAX_TOKENS_OVERRIDE" \ + --argjson fallback_context "$DEFAULT_CONTEXT_WINDOW" \ + --argjson fallback_max "$DEFAULT_MAX_TOKENS" ' + def positive_number($value; $fallback): + (try ($value | tonumber) catch $fallback) + | if type == "number" and . > 0 then . else $fallback end; + [ + (.data // .models // [])[]? + | if type == "string" then {id: .} elif type == "object" then . else empty end + | (.id // .slug // .model // "") as $id + | select(($id | type) == "string" and ($id | length) > 0) + | (if $context_override != "" + then positive_number($context_override; 0) + else positive_number((.max_input_tokens // .context_window // .contextWindow // .context_length); $fallback_context) + end) as $context + | select($context > 0) + | (if $max_override != "" + then positive_number($max_override; 0) + else ([positive_number((.max_output_tokens // .maxOutputTokens); $fallback_max), $fallback_max] | min) + end) as $output + | ((.display_name // .displayName // .name // $id) + | if type == "string" and length > 0 then . else $id end) as $name + | { + id: $id, + name: (if ($name | endswith("(Webster)")) then $name else ($name + " (Webster)") end), + reasoning: (if (.reasoning | type) == "boolean" then .reasoning else true end), + input: ["text"], + contextWindow: $context, + maxTokens: ([$output, $context] | min) + } + ] + | unique_by(.id) + | sort_by(.id) + | if length > 0 then . else error("no usable models in endpoint response") end + ' "$response_file" +} + +# fetch_webster_models KEY — update DISCOVERED_MODELS from the live endpoint. +# DISCOVERY_STATUS distinguishes authentication and empty-access failures from +# transient/unparseable responses that may safely reuse an installed catalog. +fetch_webster_models() { + local key="$1" response_file http_status normalized + response_file=$(mktemp) + DISCOVERY_STATUS="" + if ! http_status=$(printf 'Authorization: Bearer %s\n' "$key" \ + | curl -sS -o "$response_file" -w '%{http_code}' -m 20 \ + --header @- "$BASE_URL/models" 2>/dev/null); then + DISCOVERY_STATUS="unreachable" + rm -f "$response_file" + return 1 + fi + + case "$http_status" in + 2*) + if normalized=$(normalize_models_response "$response_file" 2>/dev/null); then + DISCOVERED_MODELS="$normalized" + DISCOVERY_STATUS="ok" + rm -f "$response_file" + return 0 + fi + if jq -e '((.data // .models // []) | type == "array" and length == 0)' \ + "$response_file" >/dev/null 2>&1; then + DISCOVERY_STATUS="empty" + else + DISCOVERY_STATUS="invalid" + fi + ;; + 401|403) DISCOVERY_STATUS="auth-$http_status" ;; + *) DISCOVERY_STATUS="http-$http_status" ;; + esac + rm -f "$response_file" + return 1 +} + +reuse_installed_models() { + local installed + [ -f "$MODELS_JSON" ] || return 1 + installed=$(jq -ce --arg p "$PROVIDER" ' + [.providers[$p].models[]? + | select( + (.id | type) == "string" and (.id | length) > 0 + and (.contextWindow | type) == "number" and .contextWindow > 0 + and (.maxTokens | type) == "number" and .maxTokens > 0 + )] + | unique_by(.id) + | sort_by(.id) + | if length > 0 then . else error("no installed models") end + ' "$MODELS_JSON" 2>/dev/null) || return 1 + DISCOVERED_MODELS="$installed" +} + +configure_explicit_model_without_discovery() { + [ "$MODEL_EXPLICIT" = 1 ] || return 1 + local context max + context="${CONTEXT_WINDOW_OVERRIDE:-$DEFAULT_CONTEXT_WINDOW}" + max="${MAX_TOKENS_OVERRIDE:-$DEFAULT_MAX_TOKENS}" + DISCOVERED_MODELS=$(jq -cne \ + --arg id "$MODEL" --arg ctx "$context" --arg max "$max" ' + ($ctx | tonumber) as $context + | ($max | tonumber) as $output + | select($context > 0 and $output > 0) + | [{ + id: $id, + name: ($id + " (Webster)"), + reasoning: true, + input: ["text"], + contextWindow: $context, + maxTokens: ([$output, $context] | min) + }] + ') || return 1 +} + +discover_models() { + if [ "${PI_SKIP_ENDPOINT_CHECK:-0}" = 1 ]; then + if reuse_installed_models; then + log "reusing installed Webster models (PI_SKIP_ENDPOINT_CHECK=1)" + return 0 + fi + if configure_explicit_model_without_discovery; then + warn "model discovery skipped; configuring only explicit PI_MODEL=$MODEL" + return 0 + fi + warn "cannot skip model discovery on a fresh install without PI_MODEL" + record_failure pi-models + return 1 + fi + + if fetch_webster_models "$API_KEY"; then + log "discovered $(printf '%s' "$DISCOVERED_MODELS" | jq 'length') model(s) accessible to this key" + return 0 + fi + + case "$DISCOVERY_STATUS" in + auth-*) + warn "endpoint rejected the configured key (HTTP ${DISCOVERY_STATUS#auth-}) — check the key and re-run" + record_failure pi-api-key + return 1 + ;; + empty) + warn "endpoint accepted the key but advertised no accessible models" + record_failure pi-models + return 1 + ;; + esac + + if reuse_installed_models; then + warn "model discovery failed ($DISCOVERY_STATUS); reusing the installed model list" + return 0 + fi + if configure_explicit_model_without_discovery; then + warn "model discovery failed ($DISCOVERY_STATUS); configuring only explicit PI_MODEL=$MODEL" + return 0 + fi + warn "model discovery failed ($DISCOVERY_STATUS) and there is no installed model list to reuse" + record_failure pi-models + return 1 +} + +model_is_discovered() { + printf '%s' "$DISCOVERED_MODELS" | jq -e --arg model "$1" \ + 'map(.id) | index($model) != null' >/dev/null 2>&1 +} + +select_default_model() { + local installed_default="" + if [ "$MODEL_EXPLICIT" = 1 ]; then + model_is_discovered "$MODEL" || { + warn "PI_MODEL=$MODEL is not accessible to this key" + record_failure pi-settings + return 1 + } + else + if [ -f "$SETTINGS_JSON" ]; then + installed_default=$(jq -r '.defaultModel // empty' "$SETTINGS_JSON" 2>/dev/null || true) + fi + if [ -n "$installed_default" ] && model_is_discovered "$installed_default"; then + MODEL="$installed_default" + elif model_is_discovered "$PREFERRED_MODEL"; then + MODEL="$PREFERRED_MODEL" + else + MODEL=$(printf '%s' "$DISCOVERED_MODELS" | jq -r '.[0].id') + fi + fi + [ -n "$MODEL" ] || { + warn "could not select a default Webster model" + record_failure pi-settings + return 1 + } + log "selected default model $PROVIDER/$MODEL" +} + +# Pi does not perform provider discovery itself, so the installer writes every +# model returned for this key into models.json with endpoint-derived limits. configure_models() { - log "configuring provider '$PROVIDER' in $MODELS_JSON" + log "configuring provider '$PROVIDER' with discovered models in $MODELS_JSON" + printf '%s' "$DISCOVERED_MODELS" | jq -e 'type == "array" and length > 0' >/dev/null 2>&1 || { + warn "no discovered models are available to configure" + record_failure pi-models + return 1 + } # The key goes through the environment, not argv: anyone on the box can read # a command line out of `ps`. local filter=' @@ -379,19 +592,11 @@ configure_models() { apiKey: $ENV.PI_INSTALL_API_KEY, authHeader: true, compat: { supportsDeveloperRole: false, supportsReasoningEffort: true }, - models: [ { - id: $model, - name: $model, - reasoning: true, - input: ["text"], - contextWindow: ($ctx | tonumber), - maxTokens: ($max | tonumber) - } ] + models: $models }' PI_INSTALL_API_KEY="$API_KEY" jq_edit "$MODELS_JSON" '{"providers":{}}' "$filter" \ - --arg p "$PROVIDER" --arg url "$BASE_URL" --arg model "$MODEL" \ - --arg ctx "$CONTEXT_WINDOW" --arg max "$MAX_TOKENS" \ - --argjson exclusive "$EXCLUSIVE" \ + --arg p "$PROVIDER" --arg url "$BASE_URL" \ + --argjson models "$DISCOVERED_MODELS" --argjson exclusive "$EXCLUSIVE" \ || { record_failure pi-models; return 1; } # The file holds a live credential. chmod 600 "$MODELS_JSON" 2>/dev/null || warn "could not chmod 600 $MODELS_JSON" @@ -524,27 +729,41 @@ TOKPS_EOF # usable key. A models.json whose apiKey is empty (or still the literal env # var name) makes pi fail at request time, long after this script exits. assert_pi_provider() { - if have jq && jq -e --arg p "$PROVIDER" --arg url "$BASE_URL" --arg model "$MODEL" ' + local count + if have jq && jq -e --arg p "$PROVIDER" --arg url "$BASE_URL" ' .providers[$p].baseUrl == $url and (.providers[$p].apiKey | type == "string" and length > 0) - and (.providers[$p].models | map(.id) | index($model) != null) + and (.providers[$p].models | type == "array" and length > 0) + and (.providers[$p].models + | (map(.id) | length) == (map(.id) | unique | length)) + and all(.providers[$p].models[]; + (.id | type) == "string" and (.id | length) > 0 + and (.contextWindow | type) == "number" and .contextWindow > 0 + and (.maxTokens | type) == "number" and .maxTokens > 0 + and .maxTokens <= .contextWindow) ' "$MODELS_JSON" >/dev/null 2>&1; then - log "verified provider $PROVIDER serves $MODEL in $MODELS_JSON" + count=$(jq -r --arg p "$PROVIDER" '.providers[$p].models | length' "$MODELS_JSON") + log "verified provider $PROVIDER has $count discovered model(s) in $MODELS_JSON" return 0 fi - warn "provider $PROVIDER is not configured for $MODEL in $MODELS_JSON" + warn "provider $PROVIDER has no valid discovered model list in $MODELS_JSON" record_failure pi-models return 1 } assert_pi_default_model() { - if have jq && jq -e --arg p "$PROVIDER" --arg model "$MODEL" \ - '.defaultProvider == $p and .defaultModel == $model' \ - "$SETTINGS_JSON" >/dev/null 2>&1; then - log "verified default model $PROVIDER/$MODEL in $SETTINGS_JSON" + local default_model + default_model=$(jq -r '.defaultModel // empty' "$SETTINGS_JSON" 2>/dev/null || true) + if have jq && [ -n "$default_model" ] && \ + jq -e --arg p "$PROVIDER" --arg model "$default_model" \ + '.providers[$p].models | map(.id) | index($model) != null' \ + "$MODELS_JSON" >/dev/null 2>&1 && \ + jq -e --arg p "$PROVIDER" '.defaultProvider == $p' \ + "$SETTINGS_JSON" >/dev/null 2>&1; then + log "verified default model $PROVIDER/$default_model is in the discovered model list" return 0 fi - warn "default model is not $PROVIDER/$MODEL in $SETTINGS_JSON" + warn "default model is not an installed $PROVIDER model in $SETTINGS_JSON" record_failure pi-settings return 1 } @@ -566,10 +785,9 @@ assert_pi_extension() { return 1 } -# assert_pi_endpoint — prove the key is actually accepted. This is the only -# check that catches a typo'd paste, which otherwise surfaces as a 401 on the -# user's first prompt. Network problems are a warning; an explicit auth -# rejection is a failure. +# assert_pi_endpoint — prove the key is accepted and its live model list still +# matches models.json. Network problems preserve installed state as a warning; +# authentication, empty access, and confirmed catalog drift are failures. assert_pi_endpoint() { if [ "${PI_SKIP_ENDPOINT_CHECK:-0}" = 1 ]; then log "skipping endpoint check (PI_SKIP_ENDPOINT_CHECK=1)" @@ -577,21 +795,33 @@ assert_pi_endpoint() { fi have curl || { warn "curl unavailable; skipping endpoint check"; return 0; } - local key http_status + local key installed live key=$(jq -r --arg p "$PROVIDER" '.providers[$p].apiKey // ""' "$MODELS_JSON" 2>/dev/null || echo "") [ -n "$key" ] || { warn "no API key on disk; skipping endpoint check"; return 0; } - http_status=$(curl -s -o /dev/null -w '%{http_code}' -m 20 \ - -H "Authorization: Bearer $key" "$BASE_URL/models" 2>/dev/null || echo "000") + if fetch_webster_models "$key"; then + installed=$(jq -cS --arg p "$PROVIDER" '.providers[$p].models | sort_by(.id)' \ + "$MODELS_JSON" 2>/dev/null || echo "") + live=$(printf '%s' "$DISCOVERED_MODELS" | jq -cS 'sort_by(.id)') + if [ "$installed" = "$live" ]; then + log "verified installed models match current endpoint access" + return 0 + fi + warn "installed Webster models differ from current endpoint access; re-run pi-setup.sh" + record_failure pi-models + return 1 + fi - case "$http_status" in - 2*) log "verified $BASE_URL accepts the configured key (HTTP $http_status)"; return 0 ;; - 401|403) - warn "endpoint rejected the configured key (HTTP $http_status) — check the key and re-run" + case "$DISCOVERY_STATUS" in + auth-*) + warn "endpoint rejected the configured key (HTTP ${DISCOVERY_STATUS#auth-}) — check the key and re-run" record_failure pi-api-key return 1 ;; - 000) warn "could not reach $BASE_URL; skipping endpoint check"; return 0 ;; - *) warn "unexpected response from $BASE_URL (HTTP $http_status); skipping endpoint check"; return 0 ;; + empty) + warn "endpoint accepts the configured key but advertises no accessible models" + record_failure pi-models + return 1 ;; + *) warn "could not validate live model access ($DISCOVERY_STATUS); keeping installed models"; return 0 ;; esac } @@ -635,8 +865,10 @@ main() { ensure_node || warn "Node.js install failed" install_pi || warn "pi install failed" - resolve_api_key || { summary; return; } - configure_models || warn "writing models.json failed" + resolve_api_key || { summary; return; } + discover_models || { summary; return; } + select_default_model || { summary; return; } + configure_models || warn "writing models.json failed" configure_settings || warn "writing settings.json failed" install_extension || warn "installing the tokps-session extension failed" diff --git a/test.sh b/test.sh index 3f9b48b..570932b 100755 --- a/test.sh +++ b/test.sh @@ -672,6 +672,8 @@ fi # Sourced in subshells: it defines its own log/have/record_failure helpers, # which would otherwise shadow install.sh's for the tests above. +pi_test_models='[{"id":"test-model","name":"Test Model (Webster)","reasoning":true,"input":["text"],"contextWindow":131072,"maxTokens":8192}]' + # resolve_api_key must fail rather than silently writing a keyless config when # no key is available and prompting is disabled (cron, provisioning, CI). if ( @@ -704,6 +706,7 @@ if ( export SETUP_SKIP_MAIN=1 PI_CODING_AGENT_DIR="$scratch/pi-agent" source ./pi-setup.sh API_KEY=test-key + DISCOVERED_MODELS="$pi_test_models" configure_models ) >/dev/null 2>&1; then echo "FAIL: pi configure_models unexpectedly succeeded on invalid JSON" >&2 @@ -722,6 +725,7 @@ if ! ( export SETUP_SKIP_MAIN=1 PI_CODING_AGENT_DIR="$scratch/pi-agent" source ./pi-setup.sh API_KEY=test-key + DISCOVERED_MODELS="$pi_test_models" configure_models ) >/dev/null 2>&1; then echo "FAIL: pi configure_models failed on a config with an existing provider" >&2 @@ -732,6 +736,145 @@ if ! grep -q 'other.example.com' "$scratch/pi-agent/models.json"; then exit 1 fi +# Discovery must write exactly the models returned for this key, carry through +# per-model limits, choose an accessible default when glm-5.2 is absent, and be +# byte-identical on a safe re-run. +pi_dynamic_dir="$scratch/pi-agent-dynamic" +if have python3; then + python3 - "$scratch/pi-discovery-port" >/dev/null 2>&1 <<'PYEOF' & +import http.server, json, socketserver, sys, threading + +class ScopedModels(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.headers.get("Authorization") == "Bearer empty-key": + body = json.dumps({"data": []}).encode() + self.send_response(200) + elif self.headers.get("Authorization") == "Bearer scoped-key": + body = json.dumps({"data": [ + {"id": "future-model", "max_input_tokens": 222000, "max_output_tokens": 12000}, + {"id": "access-only", "max_input_tokens": 64000, "max_output_tokens": 4096}, + {"id": "metadata-light"}, + ]}).encode() + self.send_response(200) + else: + body = b"" + self.send_response(401) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + +with socketserver.TCPServer(("127.0.0.1", 0), ScopedModels) as srv: + with open(sys.argv[1], "w") as fh: + fh.write(str(srv.server_address[1])) + threading.Timer(30, srv.shutdown).start() + srv.serve_forever() +PYEOF + pi_discovery_pid=$! + for _ in $(seq 1 50); do [ -s "$scratch/pi-discovery-port" ] && break; sleep 0.1; done + if [ -s "$scratch/pi-discovery-port" ]; then + if ! ( + export SETUP_SKIP_MAIN=1 PI_CODING_AGENT_DIR="$pi_dynamic_dir" + export PI_BASE_URL="http://127.0.0.1:$(cat "$scratch/pi-discovery-port")/v1" + source ./pi-setup.sh + API_KEY=scoped-key + discover_models + select_default_model + configure_models + configure_settings + assert_pi_provider + assert_pi_default_model + assert_pi_endpoint + ) >/dev/null 2>&1; then + echo "FAIL: pi dynamic model discovery/configuration failed" >&2 + kill "$pi_discovery_pid" 2>/dev/null || true + exit 1 + fi + if ! jq -e ' + (.providers.webster.models | map(.id)) == ["access-only", "future-model", "metadata-light"] + and (.providers.webster.models[0].contextWindow == 64000) + and (.providers.webster.models[0].maxTokens == 4096) + and (.providers.webster.models[1].contextWindow == 222000) + and (.providers.webster.models[1].maxTokens == 12000) + and (.providers.webster.models[2].contextWindow == 320000) + and (.providers.webster.models[2].maxTokens == 32768) + ' "$pi_dynamic_dir/models.json" >/dev/null || \ + [ "$(jq -r '.defaultModel' "$pi_dynamic_dir/settings.json")" != "access-only" ]; then + echo "FAIL: pi did not persist the key-scoped models, limits, and accessible default" >&2 + kill "$pi_discovery_pid" 2>/dev/null || true + exit 1 + fi + pi_dynamic_first=$(cat "$pi_dynamic_dir/models.json") + if ! ( + export SETUP_SKIP_MAIN=1 PI_CODING_AGENT_DIR="$pi_dynamic_dir" + export PI_BASE_URL="http://127.0.0.1:$(cat "$scratch/pi-discovery-port")/v1" + source ./pi-setup.sh + API_KEY=scoped-key + discover_models + select_default_model + configure_models + ) >/dev/null 2>&1 || [ "$(cat "$pi_dynamic_dir/models.json")" != "$pi_dynamic_first" ]; then + echo "FAIL: pi dynamic model discovery is not idempotent" >&2 + kill "$pi_discovery_pid" 2>/dev/null || true + exit 1 + fi + if ( + export SETUP_SKIP_MAIN=1 PI_CODING_AGENT_DIR="$scratch/pi-agent-empty" + export PI_BASE_URL="http://127.0.0.1:$(cat "$scratch/pi-discovery-port")/v1" + source ./pi-setup.sh + API_KEY=empty-key + discover_models + ) >/dev/null 2>&1; then + echo "FAIL: pi discovery accepted a key with no accessible models" >&2 + kill "$pi_discovery_pid" 2>/dev/null || true + exit 1 + fi + else + echo "WARN: skipping pi dynamic discovery tests (loopback server did not start)" >&2 + fi + kill "$pi_discovery_pid" 2>/dev/null || true + wait "$pi_discovery_pid" 2>/dev/null || true +fi + +# A transiently unreachable endpoint must preserve an already installed model +# list so an offline re-run does not erase working configuration. +if [ -f "$pi_dynamic_dir/models.json" ] && ! ( + export SETUP_SKIP_MAIN=1 PI_CODING_AGENT_DIR="$scratch/pi-agent-dynamic" + export PI_BASE_URL="http://127.0.0.1:9/v1" + source ./pi-setup.sh + API_KEY=scoped-key + discover_models + [ "$(printf '%s' "$DISCOVERED_MODELS" | jq 'length')" = 3 ] +) >/dev/null 2>&1; then + echo "FAIL: pi offline discovery did not reuse the installed model list" >&2 + exit 1 +fi + +# An explicit default must be authorized, and a fresh discovery-skipped install +# must not silently fall back to a hardcoded model. +if ( + export SETUP_SKIP_MAIN=1 PI_MODEL=not-authorized + source ./pi-setup.sh + DISCOVERED_MODELS="$pi_test_models" + select_default_model +) >/dev/null 2>&1; then + echo "FAIL: pi accepted an explicit default that was not discovered" >&2 + exit 1 +fi +if ( + export SETUP_SKIP_MAIN=1 PI_SKIP_ENDPOINT_CHECK=1 + export PI_CODING_AGENT_DIR="$scratch/pi-agent-fresh-skipped" + unset PI_MODEL + source ./pi-setup.sh + discover_models +) >/dev/null 2>&1; then + echo "FAIL: pi fresh install skipped discovery without an explicit model" >&2 + exit 1 +fi + # A key the endpoint rejects must be reported, not left for the user to hit as # a 401 on their first prompt. Served from loopback so this needs neither # network access nor a real credential. @@ -762,6 +905,7 @@ PYEOF export PI_BASE_URL="http://127.0.0.1:$(cat "$scratch/port")/v1" source ./pi-setup.sh API_KEY=wrong-key + DISCOVERED_MODELS="$pi_test_models" configure_models >/dev/null assert_pi_endpoint ) >/dev/null 2>&1; then @@ -781,6 +925,7 @@ if ( export SETUP_SKIP_MAIN=1 PI_CODING_AGENT_DIR="$scratch/pi-agent-perms" source ./pi-setup.sh API_KEY=test-key + DISCOVERED_MODELS="$pi_test_models" configure_models ) >/dev/null 2>&1; then # Same GNU-then-BSD stat ordering as the omp check above.