From 9159cee3367240441362150bd9885e874a244a47 Mon Sep 17 00:00:00 2001 From: VRWallace Date: Mon, 12 Oct 2026 21:21:16 -0500 Subject: [PATCH 1/4] Sync Bonsai 2 Hadamard ternary kernels (PTQ1_0/PQ2_0) + CUDA 13.4 & MSVC Win32 fixes --- ggml/src/ggml-cuda/argsort.cu | 2 +- ggml/src/ggml-cuda/top-k.cu | 2 +- src/llama-arch.cpp | 2 +- vendor/cpp-httplib/httplib.cpp | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-cuda/argsort.cu b/ggml/src/ggml-cuda/argsort.cu index 26af90025972..8c87ec204b34 100644 --- a/ggml/src/ggml-cuda/argsort.cu +++ b/ggml/src/ggml-cuda/argsort.cu @@ -2,7 +2,7 @@ #ifdef GGML_CUDA_USE_CUB # include -# if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 1) +# if (CCCL_MAJOR_VERSION == 3 && CCCL_MINOR_VERSION == 1) # define STRIDED_ITERATOR_AVAILABLE # include # endif diff --git a/ggml/src/ggml-cuda/top-k.cu b/ggml/src/ggml-cuda/top-k.cu index 9681cd293338..e86d44c5e365 100644 --- a/ggml/src/ggml-cuda/top-k.cu +++ b/ggml/src/ggml-cuda/top-k.cu @@ -3,7 +3,7 @@ #ifdef GGML_CUDA_USE_CUB # include -# if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 2) +# if (CCCL_MAJOR_VERSION == 3 && CCCL_MINOR_VERSION == 2) # define CUB_TOP_K_AVAILABLE # include using namespace cub; diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 4c3fcb2575a4..05ed815eb923 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -1022,7 +1022,7 @@ LLM_TN_IMPL::LLM_TN_IMPL(llm_arch arch, llm_tensor tensor, const char * suffix, std::string LLM_TN_IMPL::str() const { if (LLM_TENSOR_NAMES.find(tensor) == LLM_TENSOR_NAMES.end()) { - GGML_ABORT("unknown tensor name for tensor id %d", static_cast(tensor)); + return "unknown"; } std::string name = ::format(LLM_TENSOR_NAMES.at(tensor), bid, xid); diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp index 81cdcfe3a0c8..99760e41e46f 100644 --- a/vendor/cpp-httplib/httplib.cpp +++ b/vendor/cpp-httplib/httplib.cpp @@ -1517,8 +1517,8 @@ bool mmap::open(const char *path) { if (wpath.empty()) { return false; } hFile_ = - ::CreateFile2(wpath.c_str(), GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING, NULL); + ::CreateFileW(wpath.c_str(), GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (hFile_ == INVALID_HANDLE_VALUE) { return false; } From 0dc777158446ad19f73651107f7c2682cce969b3 Mon Sep 17 00:00:00 2001 From: VRWallace Date: Mon, 12 Oct 2026 23:53:11 -0500 Subject: [PATCH 2/4] Add WinSW service manager and Bonsai2/Gemma4 service configs (80K q8_0 KV, reasoning off) --- Bonsai2Service.xml | 12 ++ Gemma4Service.xml | 12 ++ servicemanager.ps1 | 462 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 486 insertions(+) create mode 100644 Bonsai2Service.xml create mode 100644 Gemma4Service.xml create mode 100644 servicemanager.ps1 diff --git a/Bonsai2Service.xml b/Bonsai2Service.xml new file mode 100644 index 000000000000..6e009a1b909b --- /dev/null +++ b/Bonsai2Service.xml @@ -0,0 +1,12 @@ + + Bonsai2Service + Ternary Bonsai 2 27B llama.cpp Service + Native Windows Service (WinSW) for PrismML Ternary Bonsai 2 27B (PQ2_0) with CUDA 13.4 GPU acceleration + C:\Users\vonwa\.gemini\antigravity\scratch\prism_cuda\bin\llama-server.exe + -m "C:\Users\vonwa\.gemini\antigravity\scratch\llama_gemma\models\Ternary-Bonsai-2-27B-PQ2_0.gguf" --host 0.0.0.0 --port 8080 -c 81920 -ctk q8_0 -ctv q8_0 -ngl 99 -fa on -t 16 -b 2048 -ub 1024 --reasoning off --reasoning-budget 0 --temp 0.7 --top-p 0.80 --top-k 20 --min-p 0.0 --presence-penalty 1.5 + C:\Users\vonwa\.gemini\antigravity\scratch\prism_cuda + C:\Users\vonwa\.gemini\antigravity\scratch\prism_cuda\logs + + Automatic + + diff --git a/Gemma4Service.xml b/Gemma4Service.xml new file mode 100644 index 000000000000..4488b5d1e2a0 --- /dev/null +++ b/Gemma4Service.xml @@ -0,0 +1,12 @@ + + Gemma4Service + Gemma 4 12B llama.cpp Service + Native Windows Service (WinSW) for Google Gemma 4 12B LLM inference engine with CUDA 13.4 GPU acceleration + C:\Users\vonwa\.gemini\antigravity\scratch\llama_gemma\bin\llama-server.exe + -m "C:\Users\vonwa\.gemini\antigravity\scratch\llama_gemma\models\gemma-4-12B-it-Q4_K_M.gguf" --host 0.0.0.0 --port 8081 -c 16384 -ngl 99 -fa 1 -t 16 -b 2048 -ub 512 -sps 0.5 --reasoning-budget 0 --spec-type ngram-simple + C:\Users\vonwa\.gemini\antigravity\scratch\llama_gemma + C:\Users\vonwa\.gemini\antigravity\scratch\llama_gemma\logs + + Manual + + diff --git a/servicemanager.ps1 b/servicemanager.ps1 new file mode 100644 index 000000000000..9cc2d77ad86d --- /dev/null +++ b/servicemanager.ps1 @@ -0,0 +1,462 @@ +# ============================================================================== +# Unified WinSW Service Manager (Gemma4Service & Bonsai2Service) +# Manages Native Windows Services via WinSW (Windows Service Wrapper): +# - Gemma4Service (Google Gemma 4 12B Q4_K_M) +# - Bonsai2Service (PrismML Ternary Bonsai 2 27B - PTQ1_0 or PQ2_0) +# ============================================================================== +param ( + [Parameter(Position=0, Mandatory=$false)] + [string]$Action = "menu", + + [Parameter(Position=1, Mandatory=$false)] + [string]$Target = "", + + [Parameter(Position=2, Mandatory=$false)] + [int]$CustomPort = 0 +) + +# Self-elevate to Administrator when running interactive menu so WinSW services can be controlled +$isAdmin = ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) +if (-not $isAdmin -and $Action -eq "menu") { + Write-Host "[+] Elevating to Administrator to manage WinSW Windows Services..." -ForegroundColor Cyan + Start-Process powershell.exe -ArgumentList "-NoProfile -ExecutionPolicy Bypass -File `"$PSCommandPath`"" -Verb RunAs + exit +} + +$GemmaDir = "C:\Users\vonwa\.gemini\antigravity\scratch\llama_gemma" +$BonsaiDir = "C:\Users\vonwa\.gemini\antigravity\scratch\prism_cuda" +$ModelsDir = Join-Path $GemmaDir "models" +$ConfigFile = Join-Path $GemmaDir "winsw_manager_config.json" + +$GemmaExe = Join-Path $GemmaDir "Gemma4Service.exe" +$GemmaXml = Join-Path $GemmaDir "Gemma4Service.xml" + +$BonsaiExe = Join-Path $BonsaiDir "Bonsai2Service.exe" +$BonsaiXml = Join-Path $BonsaiDir "Bonsai2Service.xml" + +$LlamaBench = Join-Path $BonsaiDir "bin\llama-bench.exe" +$LlamaCli = Join-Path $BonsaiDir "bin\llama-cli.exe" + +# Default State Config +$State = @{ + GemmaPort = 8081 + BonsaiPort = 8080 + BonsaiVariant = "PQ2_0" # PQ2_0 (6.70 GiB, Fastest Prompt/Decode) or PTQ1_0 (5.54 GiB, Lowest VRAM) + BonsaiCtx = 81920 # 32768 (32K), 65536 (64K), 81920 (80K q8_0), 131072 (128K q4_0), or 262144 (262K Max) + BonsaiKvType = "q8_0" # q8_0 (8-bit near-lossless KV up to 80K) or q4_0 (4-bit KV for 128K/262K) + BonsaiNgram = $false # OFF by default on Bonsai 2 (GDN recurrent layers do not allocate n_rs_seq for ngram-simple) +} + +function Load-Config { + if (Test-Path $ConfigFile) { + try { + $cfg = Get-Content $ConfigFile -Raw | ConvertFrom-Json + if ($cfg.GemmaPort) { $State.GemmaPort = [int]$cfg.GemmaPort } + if ($cfg.BonsaiPort) { $State.BonsaiPort = [int]$cfg.BonsaiPort } + if ($cfg.BonsaiVariant) { $State.BonsaiVariant = [string]$cfg.BonsaiVariant } + if ($cfg.BonsaiCtx) { $State.BonsaiCtx = [int]$cfg.BonsaiCtx } + if ($cfg.BonsaiKvType) { $State.BonsaiKvType = [string]$cfg.BonsaiKvType } + if ($null -ne $cfg.BonsaiNgram) { $State.BonsaiNgram = [bool]$cfg.BonsaiNgram } + return + } catch {} + } + Save-Config +} + +function Save-Config { + $State | ConvertTo-Json | Set-Content $ConfigFile -Encoding UTF8 +} + +Load-Config + +function Ensure-WinSWRegistered { + if (-not $isAdmin) { return } + $svcGemma = Get-Service -Name "Gemma4Service" -ErrorAction SilentlyContinue + if (-not $svcGemma -and (Test-Path $GemmaExe)) { + Write-Host "[+] Registering Gemma4Service via WinSW..." -ForegroundColor Cyan + & $GemmaExe install | Out-Null + } + $svcBonsai = Get-Service -Name "Bonsai2Service" -ErrorAction SilentlyContinue + if (-not $svcBonsai -and (Test-Path $BonsaiExe)) { + Write-Host "[+] Registering Bonsai2Service via WinSW..." -ForegroundColor Cyan + & $BonsaiExe install | Out-Null + Set-Service -Name "Bonsai2Service" -StartupType Manual -ErrorAction SilentlyContinue + } +} + +Ensure-WinSWRegistered + +function Update-GemmaXml { + param([string]$StartMode = "Automatic") + $modelPath = Join-Path $ModelsDir "gemma-4-12B-it-Q4_K_M.gguf" + $xml = @" + + Gemma4Service + Gemma 4 12B llama.cpp Service + Native Windows Service (WinSW) for Google Gemma 4 12B LLM inference engine with CUDA 13.4 GPU acceleration + $GemmaDir\bin\llama-server.exe + -m "$modelPath" --host 0.0.0.0 --port $($State.GemmaPort) -c 16384 -ngl 99 -fa 1 -t 16 -b 2048 -ub 512 -sps 0.5 --reasoning-budget 0 --spec-type ngram-simple + $GemmaDir + $GemmaDir\logs + + $StartMode + + +"@ + Set-Content -Path $GemmaXml -Value $xml -Encoding UTF8 +} + +function Update-BonsaiXml { + param([string]$StartMode = "Automatic") + $modelFile = "Ternary-Bonsai-2-27B-$($State.BonsaiVariant).gguf" + $modelPath = Join-Path $ModelsDir $modelFile + $specFlags = if ($State.BonsaiNgram) { " --spec-type ngram-simple --spec-ngram-simple-size-m 7" } else { "" } + $xml = @" + + Bonsai2Service + Ternary Bonsai 2 27B llama.cpp Service + Native Windows Service (WinSW) for PrismML Ternary Bonsai 2 27B ($($State.BonsaiVariant)) with CUDA 13.4 GPU acceleration + $BonsaiDir\bin\llama-server.exe + -m "$modelPath" --host 0.0.0.0 --port $($State.BonsaiPort) -c $($State.BonsaiCtx) -ctk $($State.BonsaiKvType) -ctv $($State.BonsaiKvType) -ngl 99 -fa on -t 16 -b 2048 -ub 1024 --reasoning off --reasoning-budget 0$specFlags --temp 0.7 --top-p 0.80 --top-k 20 --min-p 0.0 --presence-penalty 1.5 + $BonsaiDir + $BonsaiDir\logs + + $StartMode + + +"@ + Set-Content -Path $BonsaiXml -Value $xml -Encoding UTF8 +} + +function Get-NetworkIP { + $ips = (Get-NetIPAddress -AddressFamily IPv4 -ErrorAction SilentlyContinue | + Where-Object { $_.IPAddress -notlike "127.*" -and $_.IPAddress -notlike "169.254.*" }).IPAddress + $lan = $ips | Where-Object { $_ -like "192.168.*" } | Select-Object -First 1 + if ($lan) { return $lan } + if ($ips) { return ($ips | Select-Object -First 1) } + return "127.0.0.1" +} + +function Get-PortDetails { + param([int]$Port) + $conn = Get-NetTCPConnection -LocalPort $Port -State Listen -ErrorAction SilentlyContinue | Select-Object -First 1 + if (-not $conn) { return $null } + $health = "Warming up..." + try { + $r = Invoke-RestMethod -Uri "http://127.0.0.1:$Port/health" -TimeoutSec 1 -ErrorAction Stop + $health = "OK ($($r.status))" + } catch {} + return [PSCustomObject]@{ + PID = $conn.OwningProcess + Health = $health + } +} + +function Show-Dashboard { + param([bool]$Clear = $false) + if ($Clear) { try { Clear-Host } catch {} } + + $lanIp = Get-NetworkIP + Write-Host "================================================================================" -ForegroundColor Cyan + Write-Host " WinSW AI SERVICE MANAGER (Gemma4Service & Bonsai2Service - CUDA 13.4) " -ForegroundColor Green + Write-Host "================================================================================" -ForegroundColor Cyan + + try { + $gpu = nvidia-smi --query-gpu=name,memory.used,memory.free,memory.total --format=csv,noheader 2>$null + if ($gpu) { Write-Host " GPU VRAM : $gpu" -ForegroundColor Yellow } + } catch {} + Write-Host "--------------------------------------------------------------------------------" -ForegroundColor DarkCyan + + # 1. Gemma4Service Status + $svcG = Get-Service -Name "Gemma4Service" -ErrorAction SilentlyContinue + $gStat = if ($svcG) { $svcG.Status.ToString().ToUpper() } else { "NOT REGISTERED" } + $gPortInfo = Get-PortDetails -Port $State.GemmaPort + $gColor = if ($gStat -eq "RUNNING" -or $gPortInfo) { "Green" } else { "DarkYellow" } + + Write-Host (" [1] Gemma4Service (Gemma 4 12B Q4_K_M - 7.14 GB) | Port: {0,-5} | " -f $State.GemmaPort) -NoNewline -ForegroundColor White + if ($gPortInfo) { + Write-Host ("RUNNING (PID {0}, Health: {1})" -f $gPortInfo.PID, $gPortInfo.Health) -ForegroundColor Green + Write-Host (" -> Local: http://127.0.0.1:{0} | LAN: http://{1}:{0} | WinSW: {2}" -f $State.GemmaPort, $lanIp, $gStat) -ForegroundColor Cyan + } else { + Write-Host ("STOPPED (WinSW: {0})" -f $gStat) -ForegroundColor $gColor + } + + Write-Host "" + + # 2. Bonsai2Service Status + $svcB = Get-Service -Name "Bonsai2Service" -ErrorAction SilentlyContinue + $bStat = if ($svcB) { $svcB.Status.ToString().ToUpper() } else { "READY (Click Start)" } + $bPortInfo = Get-PortDetails -Port $State.BonsaiPort + $bSize = if ($State.BonsaiVariant -eq "PTQ1_0") { "5.95 GB" } else { "7.21 GB" } + $ctxK = [math]::Round($State.BonsaiCtx / 1024) + $ngStr = if ($State.BonsaiNgram) { "ON (m=7)" } else { "OFF" } + + Write-Host (" [2] Bonsai2Service (Ternary Bonsai 2 27B {0} - {1}) | Port: {2,-5} | " -f $State.BonsaiVariant, $bSize, $State.BonsaiPort) -NoNewline -ForegroundColor White + if ($bPortInfo) { + Write-Host ("RUNNING (PID {0}, Health: {1})" -f $bPortInfo.PID, $bPortInfo.Health) -ForegroundColor Green + Write-Host (" -> Local: http://127.0.0.1:{0} | LAN: http://{1}:{0} | WinSW: {2}" -f $State.BonsaiPort, $lanIp, $bStat) -ForegroundColor Cyan + } else { + Write-Host ("STOPPED (WinSW: {0})" -f $bStat) -ForegroundColor DarkYellow + } + Write-Host (" -> Context: {0} ({1}K, KV={2}) | N-Gram Speculative: {3} | Thinking: OFF" -f $State.BonsaiCtx, $ctxK, $State.BonsaiKvType, $ngStr) -ForegroundColor Gray + + Write-Host "================================================================================" -ForegroundColor Cyan +} + +function Stop-GemmaService { + Write-Host "[+] Stopping Gemma4Service (Port $($State.GemmaPort))..." -ForegroundColor Yellow + if (Test-Path $GemmaExe) { & $GemmaExe stop 2>$null | Out-Null } + Stop-Service -Name "Gemma4Service" -Force -ErrorAction SilentlyContinue + Set-Service -Name "Gemma4Service" -StartupType Manual -ErrorAction SilentlyContinue + $p = Get-PortDetails -Port $State.GemmaPort + if ($p) { + Stop-Process -Id $p.PID -Force -ErrorAction SilentlyContinue + & taskkill /F /PID $p.PID 2>$null | Out-Null + } + Start-Sleep -Seconds 1 + Write-Host "[SUCCESS] Gemma4Service stopped." -ForegroundColor Green +} + +function Stop-BonsaiService { + Write-Host "[+] Stopping Bonsai2Service (Port $($State.BonsaiPort))..." -ForegroundColor Yellow + if (Test-Path $BonsaiExe) { & $BonsaiExe stop 2>$null | Out-Null } + Stop-Service -Name "Bonsai2Service" -Force -ErrorAction SilentlyContinue + Set-Service -Name "Bonsai2Service" -StartupType Manual -ErrorAction SilentlyContinue + $p = Get-PortDetails -Port $State.BonsaiPort + if ($p) { + Stop-Process -Id $p.PID -Force -ErrorAction SilentlyContinue + & taskkill /F /PID $p.PID 2>$null | Out-Null + } + Start-Sleep -Seconds 1 + Write-Host "[SUCCESS] Bonsai2Service stopped." -ForegroundColor Green +} + +function Start-GemmaService { + $p = Get-PortDetails -Port $State.GemmaPort + if ($p) { + $svcG = Get-Service -Name "Gemma4Service" -ErrorAction SilentlyContinue + if (-not $svcG -or $svcG.Status -ne "Running") { + Stop-Process -Id $p.PID -Force -ErrorAction SilentlyContinue + Start-Sleep -Seconds 1 + } + } + Update-GemmaXml -StartMode "Automatic" + Ensure-WinSWRegistered + Set-Service -Name "Gemma4Service" -StartupType Automatic -ErrorAction SilentlyContinue + Write-Host "[+] Starting Gemma4Service via WinSW on Port $($State.GemmaPort)..." -ForegroundColor Green + & $GemmaExe start 2>$null | Out-Null + Start-Service -Name "Gemma4Service" -ErrorAction SilentlyContinue + Wait-ForPortHealth -Port $State.GemmaPort -Label "Gemma 4 12B" +} + +function Start-BonsaiService { + $p = Get-PortDetails -Port $State.BonsaiPort + if ($p) { + $svcB = Get-Service -Name "Bonsai2Service" -ErrorAction SilentlyContinue + if (-not $svcB -or $svcB.Status -ne "Running") { + Stop-Process -Id $p.PID -Force -ErrorAction SilentlyContinue + Start-Sleep -Seconds 1 + } + } + Update-BonsaiXml -StartMode "Automatic" + Ensure-WinSWRegistered + Set-Service -Name "Bonsai2Service" -StartupType Automatic -ErrorAction SilentlyContinue + Write-Host "[+] Starting Bonsai2Service ($($State.BonsaiVariant), -c $($State.BonsaiCtx)) via WinSW on Port $($State.BonsaiPort)..." -ForegroundColor Green + & $BonsaiExe start 2>$null | Out-Null + Start-Service -Name "Bonsai2Service" -ErrorAction SilentlyContinue + Wait-ForPortHealth -Port $State.BonsaiPort -Label "Ternary Bonsai 2 27B ($($State.BonsaiVariant))" +} + +function Wait-ForPortHealth { + param([int]$Port, [string]$Label) + Write-Host "[+] Waiting for $Label on Port $Port to finish loading into GPU VRAM..." -ForegroundColor Cyan + for ($i = 1; $i -le 20; $i++) { + Start-Sleep -Seconds 1 + try { + $r = Invoke-RestMethod -Uri "http://127.0.0.1:$Port/health" -TimeoutSec 1 -ErrorAction Stop + if ($r.status -eq "ok") { + Write-Host "[SUCCESS] $Label is ONLINE and READY on Port $Port!" -ForegroundColor Green + return + } + } catch {} + } + Write-Host "[i] $Label service started on Port $Port (still initializing weights)." -ForegroundColor Yellow +} + +function Pause-Console { + Write-Host "" + Write-Host "Press Enter to return to menu..." -ForegroundColor DarkGray + $null = Read-Host +} + +function Show-Menu { + while ($true) { + Show-Dashboard -Clear $true + $ctxK = [math]::Round($State.BonsaiCtx / 1024) + $ngStr = if ($State.BonsaiNgram) { "ON (m=7, 8-col MMVQ)" } else { "OFF" } + Write-Host " EXCLUSIVE MODE (Runs ONLY One Service for 100% Dedicated GPU VRAM & Max Speed):" -ForegroundColor Yellow + Write-Host " [1] Run ONLY Gemma 4 12B (Stops Bonsai 2, Starts Gemma4Service on Port $($State.GemmaPort))" -ForegroundColor Green + Write-Host " [2] Run ONLY Bonsai 2 27B (PTQ1_0) (Stops Gemma 4, Starts Bonsai2Service PTQ1_0 on Port $($State.BonsaiPort))" -ForegroundColor Green + Write-Host " [3] Run ONLY Bonsai 2 27B (PQ2_0) (Stops Gemma 4, Starts Bonsai2Service PQ2_0 on Port $($State.BonsaiPort))" -ForegroundColor Green + Write-Host "" + Write-Host " INDIVIDUAL & DUAL SERVICE CONTROL (WinSW):" -ForegroundColor Yellow + Write-Host " [4] Start Gemma4Service (Port $($State.GemmaPort))" -ForegroundColor Cyan + Write-Host " [5] Stop Gemma4Service (Port $($State.GemmaPort))" -ForegroundColor Cyan + Write-Host " [6] Start Bonsai2Service (Port $($State.BonsaiPort) - $($State.BonsaiVariant))" -ForegroundColor Cyan + Write-Host " [7] Stop Bonsai2Service (Port $($State.BonsaiPort))" -ForegroundColor Cyan + Write-Host " [8] Run BOTH Services Simultaneously (Gemma 4 on $($State.GemmaPort) + Bonsai 2 on $($State.BonsaiPort))" -ForegroundColor Magenta + Write-Host " [9] Shut Down BOTH Services (Free 100% GPU VRAM)" -ForegroundColor Red + Write-Host "" + Write-Host " TUNING, CONTEXT, N-GRAM & TOOLS:" -ForegroundColor Yellow + Write-Host " [K] Change Bonsai 2 Context Size (Current: $($State.BonsaiCtx) / ${ctxK}K, KV=$($State.BonsaiKvType) -> 32K / 64K / 128K / 262K Max)" -ForegroundColor White + Write-Host " [N] Toggle N-Gram Speculative (Current: $ngStr)" -ForegroundColor White + Write-Host " [P] Assign / Swap Ports for Gemma4Service ($($State.GemmaPort)) & Bonsai2Service ($($State.BonsaiPort))" -ForegroundColor White + Write-Host " [V] Toggle Bonsai 2 Variant (Current: $($State.BonsaiVariant) -> PTQ1_0 5.95GB vs PQ2_0 7.21GB)" -ForegroundColor White + Write-Host " [B] Run GPU Speed Benchmark (llama-bench)" -ForegroundColor Gray + Write-Host " [C] Launch Interactive CLI Chat" -ForegroundColor Gray + Write-Host " [R] Refresh Status [0 / Q] Exit" -ForegroundColor Gray + Write-Host "" + + $choice = (Read-Host "Select Option").Trim().ToUpper() + switch ($choice) { + "1" { + Stop-BonsaiService + Start-GemmaService + Pause-Console + } + "2" { + $State.BonsaiVariant = "PTQ1_0" + Save-Config + Stop-GemmaService + Stop-BonsaiService + Start-BonsaiService + Pause-Console + } + "3" { + $State.BonsaiVariant = "PQ2_0" + Save-Config + Stop-GemmaService + Stop-BonsaiService + Start-BonsaiService + Pause-Console + } + "4" { Start-GemmaService; Pause-Console } + "5" { Stop-GemmaService; Pause-Console } + "6" { Start-BonsaiService; Pause-Console } + "7" { Stop-BonsaiService; Pause-Console } + "8" { + if ($State.GemmaPort -eq $State.BonsaiPort) { + Write-Host "[!] Both services were set to Port $($State.GemmaPort). Assigning Gemma=8081, Bonsai=8080..." -ForegroundColor Yellow + $State.GemmaPort = 8081 + $State.BonsaiPort = 8080 + Save-Config + } + Start-GemmaService + Start-BonsaiService + Pause-Console + } + "9" { + Stop-GemmaService + Stop-BonsaiService + Get-Process -Name "llama-server" -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue + Write-Host "[SUCCESS] Both services stopped and 100% VRAM freed." -ForegroundColor Green + Pause-Console + } + "K" { + Write-Host "" + Write-Host "Select Bonsai 2 Context Window Size (RTX 4070 Ti 12 GB VRAM):" -ForegroundColor Cyan + Write-Host " [1] 32,768 ( 32K) - q8_0 KV Cache (~1.0 GiB KV VRAM, Fastest Short/Med Context)" + Write-Host " [2] 65,536 ( 64K) - q8_0 KV Cache (~2.1 GiB KV VRAM, 8-bit High Precision)" + Write-Host " [3] 81,920 ( 80K) - q8_0 KV Cache (~2.6 GiB KV VRAM, Recommended Max 8-bit Precision for PQ2_0)" + Write-Host " [4] 131,072 (128K) - q4_0 KV Cache (~2.25 GiB KV VRAM, 4-bit Ultra-Long Context)" + Write-Host " [5] 262,144 (262K) - q4_0 KV Cache (~4.50 GiB KV VRAM, Full Model Max - Best with PTQ1_0)" + $cSel = (Read-Host "Choose [1-5]").Trim() + switch ($cSel) { + "1" { $State.BonsaiCtx = 32768; $State.BonsaiKvType = "q8_0" } + "2" { $State.BonsaiCtx = 65536; $State.BonsaiKvType = "q8_0" } + "3" { $State.BonsaiCtx = 81920; $State.BonsaiKvType = "q8_0" } + "4" { $State.BonsaiCtx = 131072; $State.BonsaiKvType = "q4_0" } + "5" { + $State.BonsaiCtx = 262144 + $State.BonsaiKvType = "q4_0" + if ($State.BonsaiVariant -eq "PQ2_0") { + Write-Host "[i] Note: 262K context uses ~4.5 GiB KV cache. Switching variant to PTQ1_0 (5.54 GiB) is recommended if VRAM is tight." -ForegroundColor Yellow + } + } + } + Save-Config + Update-BonsaiXml + Write-Host "[SUCCESS] Updated Bonsai2Service context to $($State.BonsaiCtx) (-ctk $($State.BonsaiKvType) -ctv $($State.BonsaiKvType))!" -ForegroundColor Green + Write-Host "[i] Select [3] or [2] from the menu to restart Bonsai2Service with the new context." -ForegroundColor Yellow + Pause-Console + } + "N" { + $State.BonsaiNgram = -not $State.BonsaiNgram + Save-Config + Update-BonsaiXml + $st = if ($State.BonsaiNgram) { "ENABLED (--spec-type ngram-simple --spec-ngram-simple-size-m 7)" } else { "DISABLED" } + Write-Host "[SUCCESS] N-Gram Speculative Decoding is now $st." -ForegroundColor Green + Write-Host "[i] Select [3] or [2] from the menu to restart Bonsai2Service." -ForegroundColor Yellow + Pause-Console + } + "P" { + Write-Host "" + Write-Host "Quick Preset: Type 'SWAP' to swap ports ($($State.GemmaPort) <-> $($State.BonsaiPort)), or enter custom port numbers:" -ForegroundColor Cyan + $inG = (Read-Host "Enter Port for Gemma4Service (Current: $($State.GemmaPort), or type SWAP)").Trim().ToUpper() + if ($inG -eq "SWAP") { + $tmp = $State.GemmaPort + $State.GemmaPort = $State.BonsaiPort + $State.BonsaiPort = $tmp + Write-Host "[+] Swapped Ports! Gemma4Service = $($State.GemmaPort), Bonsai2Service = $($State.BonsaiPort)" -ForegroundColor Green + } else { + if ($inG -match "^\d+$") { $State.GemmaPort = [int]$inG } + $inB = (Read-Host "Enter Port for Bonsai2Service (Current: $($State.BonsaiPort))").Trim() + if ($inB -match "^\d+$") { $State.BonsaiPort = [int]$inB } + } + Save-Config + Update-GemmaXml + Update-BonsaiXml + Write-Host "[SUCCESS] Saved port configuration to Gemma4Service.xml and Bonsai2Service.xml!" -ForegroundColor Green + Write-Host "[i] Restart any running service from the menu for the new port to take effect." -ForegroundColor Yellow + Pause-Console + } + "V" { + $State.BonsaiVariant = if ($State.BonsaiVariant -eq "PTQ1_0") { "PQ2_0" } else { "PTQ1_0" } + Save-Config + Update-BonsaiXml + Write-Host "[SUCCESS] Switched Bonsai2Service variant to $($State.BonsaiVariant)." -ForegroundColor Green + Pause-Console + } + "B" { + $sel = (Read-Host "Benchmark [1] Gemma 4 12B, [2] Bonsai 2 PTQ1_0, [3] Bonsai 2 PQ2_0").Trim() + $file = switch ($sel) { + "1" { Join-Path $ModelsDir "gemma-4-12B-it-Q4_K_M.gguf" } + "3" { Join-Path $ModelsDir "Ternary-Bonsai-2-27B-PQ2_0.gguf" } + default { Join-Path $ModelsDir "Ternary-Bonsai-2-27B-PTQ1_0.gguf" } + } + & $LlamaBench -m $file -p 512 -n 128 -ngl 99 -fa on -r 3 + Pause-Console + } + "C" { + $sel = (Read-Host "Chat with [1] Gemma 4 12B, [2] Bonsai 2 PTQ1_0, [3] Bonsai 2 PQ2_0").Trim() + $file = switch ($sel) { + "1" { Join-Path $ModelsDir "gemma-4-12B-it-Q4_K_M.gguf" } + "3" { Join-Path $ModelsDir "Ternary-Bonsai-2-27B-PQ2_0.gguf" } + default { Join-Path $ModelsDir "Ternary-Bonsai-2-27B-PTQ1_0.gguf" } + } + & $LlamaCli -m $file -ngl 99 -c 16384 -fa on -cnv + Pause-Console + } + "R" { continue } + "0" { return } + "Q" { return } + "EXIT" { return } + } + } +} + +switch ($Action.ToLower()) { + "status" { Show-Dashboard -Clear $false } + "stopall" { Stop-GemmaService; Stop-BonsaiService } + default { Show-Menu } +} From 28a2d81afb171a65ebda5415370d0cf4d72339d9 Mon Sep 17 00:00:00 2001 From: VRWallace Date: Tue, 13 Oct 2026 00:00:10 -0500 Subject: [PATCH 3/4] Replace presence-penalty 1.5 with repeat-penalty, frequency-penalty, and DRY anti-repetition sampler --- Bonsai2Service.xml | 2 +- servicemanager.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Bonsai2Service.xml b/Bonsai2Service.xml index 6e009a1b909b..a6a5802e59a4 100644 --- a/Bonsai2Service.xml +++ b/Bonsai2Service.xml @@ -3,7 +3,7 @@ Ternary Bonsai 2 27B llama.cpp Service Native Windows Service (WinSW) for PrismML Ternary Bonsai 2 27B (PQ2_0) with CUDA 13.4 GPU acceleration C:\Users\vonwa\.gemini\antigravity\scratch\prism_cuda\bin\llama-server.exe - -m "C:\Users\vonwa\.gemini\antigravity\scratch\llama_gemma\models\Ternary-Bonsai-2-27B-PQ2_0.gguf" --host 0.0.0.0 --port 8080 -c 81920 -ctk q8_0 -ctv q8_0 -ngl 99 -fa on -t 16 -b 2048 -ub 1024 --reasoning off --reasoning-budget 0 --temp 0.7 --top-p 0.80 --top-k 20 --min-p 0.0 --presence-penalty 1.5 + -m "C:\Users\vonwa\.gemini\antigravity\scratch\llama_gemma\models\Ternary-Bonsai-2-27B-PQ2_0.gguf" --host 0.0.0.0 --port 8080 -c 81920 -ctk q8_0 -ctv q8_0 -ngl 99 -fa on -t 16 -b 2048 -ub 1024 --reasoning off --reasoning-budget 0 --temp 0.7 --top-p 0.80 --top-k 20 --min-p 0.0 --repeat-penalty 1.10 --frequency-penalty 0.3 --presence-penalty 0.2 --dry-multiplier 0.8 C:\Users\vonwa\.gemini\antigravity\scratch\prism_cuda C:\Users\vonwa\.gemini\antigravity\scratch\prism_cuda\logs diff --git a/servicemanager.ps1 b/servicemanager.ps1 index 9cc2d77ad86d..ffa66cf752c2 100644 --- a/servicemanager.ps1 +++ b/servicemanager.ps1 @@ -117,7 +117,7 @@ function Update-BonsaiXml { Ternary Bonsai 2 27B llama.cpp Service Native Windows Service (WinSW) for PrismML Ternary Bonsai 2 27B ($($State.BonsaiVariant)) with CUDA 13.4 GPU acceleration $BonsaiDir\bin\llama-server.exe - -m "$modelPath" --host 0.0.0.0 --port $($State.BonsaiPort) -c $($State.BonsaiCtx) -ctk $($State.BonsaiKvType) -ctv $($State.BonsaiKvType) -ngl 99 -fa on -t 16 -b 2048 -ub 1024 --reasoning off --reasoning-budget 0$specFlags --temp 0.7 --top-p 0.80 --top-k 20 --min-p 0.0 --presence-penalty 1.5 + -m "$modelPath" --host 0.0.0.0 --port $($State.BonsaiPort) -c $($State.BonsaiCtx) -ctk $($State.BonsaiKvType) -ctv $($State.BonsaiKvType) -ngl 99 -fa on -t 16 -b 2048 -ub 1024 --reasoning off --reasoning-budget 0$specFlags --temp 0.7 --top-p 0.80 --top-k 20 --min-p 0.0 --repeat-penalty 1.10 --frequency-penalty 0.3 --presence-penalty 0.2 --dry-multiplier 0.8 $BonsaiDir $BonsaiDir\logs From 6754cbaf4736cf61bd2d365a8a7d65f9f7076071 Mon Sep 17 00:00:00 2001 From: VRWallace Date: Tue, 13 Oct 2026 00:06:43 -0500 Subject: [PATCH 4/4] Fix WinSW stop/start race condition when restarting active llama-server --- servicemanager.ps1 | 50 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/servicemanager.ps1 b/servicemanager.ps1 index ffa66cf752c2..1cd3a0a1b197 100644 --- a/servicemanager.ps1 +++ b/servicemanager.ps1 @@ -205,37 +205,51 @@ function Show-Dashboard { function Stop-GemmaService { Write-Host "[+] Stopping Gemma4Service (Port $($State.GemmaPort))..." -ForegroundColor Yellow - if (Test-Path $GemmaExe) { & $GemmaExe stop 2>$null | Out-Null } - Stop-Service -Name "Gemma4Service" -Force -ErrorAction SilentlyContinue - Set-Service -Name "Gemma4Service" -StartupType Manual -ErrorAction SilentlyContinue $p = Get-PortDetails -Port $State.GemmaPort + Set-Service -Name "Gemma4Service" -StartupType Manual -ErrorAction SilentlyContinue if ($p) { - Stop-Process -Id $p.PID -Force -ErrorAction SilentlyContinue & taskkill /F /PID $p.PID 2>$null | Out-Null + Stop-Process -Id $p.PID -Force -ErrorAction SilentlyContinue + } + if (Test-Path $GemmaExe) { & $GemmaExe stop 2>$null | Out-Null } + Stop-Service -Name "Gemma4Service" -Force -ErrorAction SilentlyContinue + for ($i = 0; $i -lt 15; $i++) { + $svc = Get-Service -Name "Gemma4Service" -ErrorAction SilentlyContinue + if (-not $svc -or $svc.Status -eq "Stopped") { break } + Start-Sleep -Milliseconds 500 } - Start-Sleep -Seconds 1 Write-Host "[SUCCESS] Gemma4Service stopped." -ForegroundColor Green } function Stop-BonsaiService { Write-Host "[+] Stopping Bonsai2Service (Port $($State.BonsaiPort))..." -ForegroundColor Yellow - if (Test-Path $BonsaiExe) { & $BonsaiExe stop 2>$null | Out-Null } - Stop-Service -Name "Bonsai2Service" -Force -ErrorAction SilentlyContinue - Set-Service -Name "Bonsai2Service" -StartupType Manual -ErrorAction SilentlyContinue $p = Get-PortDetails -Port $State.BonsaiPort + Set-Service -Name "Bonsai2Service" -StartupType Manual -ErrorAction SilentlyContinue if ($p) { - Stop-Process -Id $p.PID -Force -ErrorAction SilentlyContinue & taskkill /F /PID $p.PID 2>$null | Out-Null + Stop-Process -Id $p.PID -Force -ErrorAction SilentlyContinue + } + if (Test-Path $BonsaiExe) { & $BonsaiExe stop 2>$null | Out-Null } + Stop-Service -Name "Bonsai2Service" -Force -ErrorAction SilentlyContinue + for ($i = 0; $i -lt 15; $i++) { + $svc = Get-Service -Name "Bonsai2Service" -ErrorAction SilentlyContinue + if (-not $svc -or $svc.Status -eq "Stopped") { break } + Start-Sleep -Milliseconds 500 } - Start-Sleep -Seconds 1 Write-Host "[SUCCESS] Bonsai2Service stopped." -ForegroundColor Green } function Start-GemmaService { + for ($i = 0; $i -lt 10; $i++) { + $svc = Get-Service -Name "Gemma4Service" -ErrorAction SilentlyContinue + if (-not $svc -or $svc.Status -ne "StopPending") { break } + Start-Sleep -Milliseconds 500 + } $p = Get-PortDetails -Port $State.GemmaPort if ($p) { $svcG = Get-Service -Name "Gemma4Service" -ErrorAction SilentlyContinue if (-not $svcG -or $svcG.Status -ne "Running") { + & taskkill /F /PID $p.PID 2>$null | Out-Null Stop-Process -Id $p.PID -Force -ErrorAction SilentlyContinue Start-Sleep -Seconds 1 } @@ -244,16 +258,25 @@ function Start-GemmaService { Ensure-WinSWRegistered Set-Service -Name "Gemma4Service" -StartupType Automatic -ErrorAction SilentlyContinue Write-Host "[+] Starting Gemma4Service via WinSW on Port $($State.GemmaPort)..." -ForegroundColor Green - & $GemmaExe start 2>$null | Out-Null Start-Service -Name "Gemma4Service" -ErrorAction SilentlyContinue + $svcG = Get-Service -Name "Gemma4Service" -ErrorAction SilentlyContinue + if (-not $svcG -or $svcG.Status -ne "Running") { + & $GemmaExe start 2>$null | Out-Null + } Wait-ForPortHealth -Port $State.GemmaPort -Label "Gemma 4 12B" } function Start-BonsaiService { + for ($i = 0; $i -lt 10; $i++) { + $svc = Get-Service -Name "Bonsai2Service" -ErrorAction SilentlyContinue + if (-not $svc -or $svc.Status -ne "StopPending") { break } + Start-Sleep -Milliseconds 500 + } $p = Get-PortDetails -Port $State.BonsaiPort if ($p) { $svcB = Get-Service -Name "Bonsai2Service" -ErrorAction SilentlyContinue if (-not $svcB -or $svcB.Status -ne "Running") { + & taskkill /F /PID $p.PID 2>$null | Out-Null Stop-Process -Id $p.PID -Force -ErrorAction SilentlyContinue Start-Sleep -Seconds 1 } @@ -262,8 +285,11 @@ function Start-BonsaiService { Ensure-WinSWRegistered Set-Service -Name "Bonsai2Service" -StartupType Automatic -ErrorAction SilentlyContinue Write-Host "[+] Starting Bonsai2Service ($($State.BonsaiVariant), -c $($State.BonsaiCtx)) via WinSW on Port $($State.BonsaiPort)..." -ForegroundColor Green - & $BonsaiExe start 2>$null | Out-Null Start-Service -Name "Bonsai2Service" -ErrorAction SilentlyContinue + $svcB = Get-Service -Name "Bonsai2Service" -ErrorAction SilentlyContinue + if (-not $svcB -or $svcB.Status -ne "Running") { + & $BonsaiExe start 2>$null | Out-Null + } Wait-ForPortHealth -Port $State.BonsaiPort -Label "Ternary Bonsai 2 27B ($($State.BonsaiVariant))" }