Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 72 additions & 0 deletions backend/disk_offload.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""Opt-in disk backing for the existing partial-loading cache."""
import json
import os
from pathlib import Path
import shutil
import sys
import tempfile
import time
from urllib.request import Request, urlopen


def confirm_disk_offload(model, required_bytes, timeout=180):
base = os.environ.get("SSUI_SERVER_URL", "http://localhost:7422").rstrip("/")

def call(path, data=None):
body = None if data is None else json.dumps(data).encode()
request = Request(base + "/api/resources/" + path, data=body,
headers={"Content-Type": "application/json"})
with urlopen(request, timeout=5) as response:
return json.load(response)

try:
key = call("disk-requests", {"model": str(model), "required_bytes": required_bytes})["id"]
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
answer = call("disk-requests/" + key)["approved"]
if answer is not None:
if answer:
return
raise MemoryError("用户拒绝磁盘 offload,已停止模型加载。")
time.sleep(.5)
except MemoryError:
raise
except Exception as exc:
raise MemoryError("内存不足,无法取得磁盘 offload 授权;已停止加载。") from exc
raise MemoryError("磁盘 offload 确认超时,已停止加载。")


def disk_back_model(model, required_bytes):
"""Own the backing directory for as long as the cache wrapper owns its tensors.

mmap uses OS paging; it does not eliminate activation/working memory requirements.
Only the state dict written by this process is loaded, with weights_only=True.
"""
import torch
directory = new_offload_directory(required_bytes)
try:
path = Path(directory.name) / "weights.pt"
state = model.state_dict()
if any(t.device.type != "cpu" for t in state.values()):
raise RuntimeError("Disk backing requires a CPU model before device loading")
torch.save(state, path)
mapped = torch.load(path, mmap=True, weights_only=True, map_location="cpu")
model.load_state_dict(mapped, assign=True)
return directory
except Exception:
directory.cleanup()
raise


def new_offload_directory(required_bytes):
if sys.platform == "win32":
cache = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
elif sys.platform == "darwin":
cache = Path.home() / "Library" / "Caches"
else:
cache = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
root = Path(os.environ.get("SSUI_OFFLOAD_DIR", cache / "SSUI" / "offload"))
root.mkdir(parents=True, exist_ok=True)
if shutil.disk_usage(root).free < required_bytes + 2**30:
raise MemoryError("磁盘可用空间不足,无法 offload(需额外预留 1 GiB)。")
return tempfile.TemporaryDirectory(prefix="ssui-offload-", dir=root, ignore_cleanup_errors=True)
20 changes: 18 additions & 2 deletions backend/model_manager/load/model_cache/model_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,16 +180,32 @@ def put(self, key: str, model: AnyModel) -> None:
# well.
running_with_cuda = self._execution_device.type == "cuda"

backing = None
keep_ram_copy = self._keep_ram_copy_of_weights
if isinstance(model, torch.nn.Module) and running_with_cuda and self._enable_partial_loading:
from backend.resource_monitor import snapshot, placement
from backend.disk_offload import confirm_disk_offload, disk_back_model
resources = snapshot()
# The model is already resident: add its bytes back when judging host capacity.
resources["ram"]["available"] += size
mode = placement(size, resources, self._execution_device.index or 0,
int(self._execution_device_working_mem_gb * GB))
if mode == "disk_confirmation":
confirm_disk_offload(key, size)
backing = disk_back_model(model, size)
keep_ram_copy = True

# Wrap model.
if isinstance(model, torch.nn.Module) and running_with_cuda and self._enable_partial_loading:
wrapped_model = CachedModelWithPartialLoad(
model, self._execution_device, keep_ram_copy=self._keep_ram_copy_of_weights
model, self._execution_device, keep_ram_copy=keep_ram_copy
)
else:
wrapped_model = CachedModelOnlyFullLoad(
model, self._execution_device, size, keep_ram_copy=self._keep_ram_copy_of_weights
model, self._execution_device, size, keep_ram_copy=keep_ram_copy
)

wrapped_model._disk_backing = backing
cache_record = CacheRecord(key=key, cached_model=wrapped_model)
self._cached_models[key] = cache_record
self._cache_stack.append(key)
Expand Down
35 changes: 35 additions & 0 deletions backend/resource_monitor.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
"""Machine resources and conservative model placement decisions (all sizes are bytes)."""

GiB = 2**30


def snapshot():
# Schema-only imports must not require optional monitoring dependencies.
import psutil
ram = psutil.virtual_memory()
result = {"ram": {"total": ram.total, "available": ram.available,
"used": ram.total - ram.available}, "gpus": [], "gpu_error": None}
try:
import torch
if torch.cuda.is_available():
for index in range(torch.cuda.device_count()):
free, total = torch.cuda.mem_get_info(index)
result["gpus"].append({"index": index, "name": torch.cuda.get_device_name(index),
"total": total, "available": free, "used": total - free})
except Exception as exc:
result["gpu_error"] = str(exc)
return result


def placement(weight_bytes, resources, device_index=0, working_bytes=3 * GiB):
"""Reserve runtime headroom; unknown VRAM is never treated as unlimited."""
if weight_bytes < 0 or working_bytes < 0:
raise ValueError("Memory estimates must be non-negative")
reserve = max(GiB, int(resources["ram"]["total"] * .1))
# This cache retains a CPU copy even when all weights fit on the GPU.
if weight_bytes + reserve > resources["ram"]["available"]:
return "disk_confirmation"
gpu = next((g for g in resources["gpus"] if g["index"] == device_index), None)
if gpu and weight_bytes + working_bytes <= gpu["available"]:
return "device"
return "cpu_offload"
176 changes: 176 additions & 0 deletions backend/runtime_offload.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
"""Shared placement policy for extension model loaders and independent runners."""
from dataclasses import dataclass
from functools import wraps
from pathlib import Path
import torch

from backend.resource_monitor import GiB, placement, snapshot
from backend.disk_offload import confirm_disk_offload, disk_back_model

_WEIGHT_SUFFIXES = {'.safetensors', '.pt', '.pth', '.bin', '.ckpt'}


def weight_size(paths):
"""Conservative on-disk estimate; deduplicate shards and exclude tokenizer/config files."""
if isinstance(paths, (str, Path)):
paths = [paths]
files = set()
for value in paths:
path = Path(value)
if path.is_dir():
files.update(p.resolve() for p in path.rglob('*') if p.suffix in _WEIGHT_SUFFIXES and p.is_file())
elif path.is_file() and path.suffix in _WEIGHT_SUFFIXES:
files.add(path.resolve())
return sum(p.stat().st_size for p in files)


@dataclass
class LoadPlan:
mode: str
device: str
estimated_bytes: int


def plan_load(label, paths=(), device='cuda', requested=None, estimated_bytes=None):
"""Ask before CPU staging starts. A request authorizes only this load invocation."""
size = weight_size(paths) if estimated_bytes is None else estimated_bytes
if size <= 0:
raise MemoryError(f'{label}: 无法估算模型权重大小,已停止自动加载。')
target = torch.device(device)
resources = snapshot()
index = target.index if target.index is not None else (torch.cuda.current_device() if target.type == 'cuda' and torch.cuda.is_available() else 0)
mode = placement(size, resources, device_index=index)
if requested == 'disk' or mode == 'disk_confirmation':
confirm_disk_offload(label, size)
mode = 'disk_offload'
elif requested == 'cpu' or target.type != 'cuda':
mode = 'cpu_offload'
return LoadPlan(mode, str(target), size)


def configure_model(model, plan):
"""Use Accelerate layer hooks; keep the execution device separate from meta weights.

Disk mode supplies mmap-backed CPU tensors to the same hooks. Offload weights are
never moved en masse by pipeline stage transitions. Custom tensor subclasses are
rejected before hooks are installed: silently dequantizing them is unsafe.
"""
if getattr(model, '_ssui_residency', None) is not None:
return model
if plan.mode == 'device':
model.to(plan.device)
return model
if any(type(p) is not torch.nn.Parameter for p in model.parameters()):
raise MemoryError('该模型的自定义量化权重不支持自动 offload,请使用框架原生 offload。')
from accelerate import cpu_offload
model.cpu()
backing = None
try:
size = sum(t.numel() * t.element_size() for t in model.state_dict().values())
if plan.mode == 'disk_offload':
backing = disk_back_model(model, size)
# Some attention implementations read a child's weights directly.
preload = ['MultiheadAttention']
cpu_offload(model, execution_device=torch.device(plan.device), offload_buffers=False,
state_dict=model.state_dict(), preload_module_classes=preload)
# Accelerate's forward wrapper does not run post_forward after an exception.
# PyTorch always_call hooks release the failing layer and its ancestors.
handles = []
def release_on_failure(module, args, output):
if output is None and hasattr(module, '_hf_hook'):
module._hf_hook.post_forward(module, None)
for module in model.modules():
if hasattr(module, '_hf_hook'):
handles.append(module.register_forward_hook(release_on_failure, always_call=True))
model._ssui_failure_hooks = handles
model._ssui_plan = plan
model._ssui_execution_device = torch.device(plan.device)
model._ssui_residency = plan.mode
model._ssui_disk_backing = backing
except Exception:
# Remove partially installed hooks and restore CPU weights before cleaning files.
from accelerate.hooks import remove_hook_from_module
remove_hook_from_module(model, recurse=True)
if backing is not None:
backing.cleanup()
raise
return model


def is_managed(model):
return getattr(model, '_ssui_residency', None) in ('cpu_offload', 'disk_offload')


def remote_weight_size(repo_id, prefixes=()):
"""Read Hub file metadata without downloading the checkpoint to RAM."""
local = Path(repo_id)
if local.exists():
return weight_size(local)
from huggingface_hub import HfApi
info = HfApi().model_info(repo_id, files_metadata=True)
files = [f for f in info.siblings if Path(f.rfilename).suffix in _WEIGHT_SUFFIXES
and (not prefixes or any(f.rfilename.startswith(p + '/') for p in prefixes))]
if not files or any(f.size is None for f in files):
raise MemoryError(f'{repo_id}: 无法读取权重大小,已停止自动加载。')
return sum(f.size for f in files)


def materialize_for_update(method):
"""Temporarily restore managed weights for existing DiffSynth LoRA editors."""
@wraps(method)
def wrapped(self, *args, **kwargs):
from accelerate.hooks import remove_hook_from_module
managed = [(m, m._ssui_plan, getattr(m, '_ssui_disk_backing', None))
for m in self.model if is_managed(m) and hasattr(m, "_ssui_plan")]
for model, plan, backing in managed:
for handle in model._ssui_failure_hooks:
handle.remove()
remove_hook_from_module(model, recurse=True)
model._ssui_residency = None
try:
return method(self, *args, **kwargs)
finally:
for model, plan, backing in managed:
configure_model(model, plan)
if backing is not None:
backing.cleanup()
return wrapped


def configure_pending(model, device, force_cpu=False):
"""Finalize after LoRA/quantization; checkpoint loaders only stage on CPU."""
pending = getattr(model, '_ssui_load_plan', None)
if pending is None or is_managed(model):
return is_managed(model)
from dataclasses import replace
plan = replace(pending, device=str(device))
if force_cpu and plan.mode == 'device':
plan = replace(plan, mode='cpu_offload')
if any('float8' in str(p.dtype) for p in model.parameters()):
if plan.mode == 'disk_offload':
raise MemoryError('FP8 存储使用框架原生精度转换;自动磁盘卸载请改用 none/int8。')
del model._ssui_load_plan
return False
if getattr(model, '_ssui_int8', False):
# Native packed Linear stores weights in buffers and exposes a computed
# weight property. Keep its native method/generator residency wrapper.
from ssui.model_memory import offload_component
model.cpu()
if plan.mode != 'device':
backing = disk_back_model(model, sum(t.numel()*t.element_size() for t in model.state_dict().values())) if plan.mode == 'disk_offload' else None
offload_component(model, device, cpu_state=model.state_dict() if backing else None)
model._ssui_disk_backing = backing
model._ssui_residency = plan.mode
model._ssui_execution_device = torch.device(device)
else:
model.to(device)
else:
configure_model(model, plan)
del model._ssui_load_plan
return is_managed(model)


def finalize_pipeline(pipe):
for module in pipe.children():
configure_pending(module, pipe.device)
return any(is_managed(m) for m in pipe.children())
57 changes: 57 additions & 0 deletions doc/ResourceMonitoring.zh.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# 内存、显存和模型 offload

功能 UI 顶部每 2 秒显示服务端机器的系统内存与各 CUDA 显卡的已用 / 总量。
内存按 total - available 计算;CUDA 显存按驱动 free / total 计算,包含其他进程占用。
未检测到 CUDA 与读取失败分别显示。Apple 共享内存设备当前只显示系统内存,不提供独立显存读数。
远程服务时显示服务端机器,非浏览器机器。

图片模型的 ModelLoaderService 和 load_model_from_path 已启用现有 CUDA 部分加载:
缓存会清理未锁定模型,保留默认 3 GiB 执行空间,按剩余显存加载权重,其余留在 CPU,
由现有自定义层按需转移。保留空间是估算,不保证任何分辨率和批次都不会 OOM。

进入缓存时发现主存余量不足(预留至少 1 GiB 或总内存 10%),会发起一次性磁盘确认。
允许后将 CPU 权重写入临时文件并 mmap 映射,仍使用同一部分加载与卸载路径。
拒绝、180 秒未答复、服务端不可达或磁盘空间不足都会停止加载。
磁盘至少需要权重大小加 1 GiB 可用空间;模型缓存释放后临时目录随对象回收。
进程异常退出可能留下 ssui-offload-* 临时目录,可在关闭应用后清理。

执行器默认连接 http://localhost:7422,通过 SSUI_SERVER_URL 指定其他服务端地址。
SSUI_OFFLOAD_DIR 可指定磁盘缓存根目录;默认为用户缓存目录下的 SSUI/offload。
请把它放在有足够空间的真实磁盘上:Linux /tmp 可能是 tmpfs,无法缓解主存不足。
授权仅适用于本次请求,不会记住为永久许可。

## 当前覆盖范围

| 入口 | 自动 CPU offload | 经确认的磁盘 offload |
| --- | --- | --- |
| 图片 ModelCache | 现有 CUDA 部分加载 | CPU 权重 mmap |
| 视频 Wan / Wan2 / Wan2.2、CogVideo、Hunyuan、AnimateDiff | DiffSynth 加载前预算 + Accelerate 逐层钩子 | mmap CPU 权重 + 同一逐层钩子 |
| TRELLIS 3D | 每个组件加载前预算 + 逐层钩子,执行设备独立保存 | 同一授权与 mmap 路径 |
| LTX 独立进程 | 主进程预检后设置原生 --offload cpu | 原生 --offload disk,仅授权后传入 |
| H3 独立进程 | Hub 文件大小预检 + ComponentsManager CPU offload / int8 组卸载 | 检查 diffusers 原生磁盘组卸载能力后启用 |

视频按整个模型组估算后再加载首个组件;TRELLIS 逐个组件检查实时余量。
本地权重文件统计会去重,目录仅统计权重文件;仓库未提供可靠大小时停止自动加载。
H3 使用 Hub 文件大小元数据,不会为估算预先把权重读进 RAM。
逐层钩子与原有阶段 .to/.cpu、Wan 显存管理保持互斥;VAE 的执行设备不从 meta 权重推断。
加载器先在 CPU 上完成 LoRA / INT8 转换,再安装卸载策略;原生 INT8 保留组件方法和生成器的驻留包装。
FP8 保留框架原生存储/计算精度转换;当前自动磁盘路径对 FP8 明确拒绝,请使用 none/int8。
DiffSynth 的 LoRA 更新会暂时恢复 CPU 权重,更新后重新安装同一卸载模式;更新仍可能产生内存峰值。
推理异常会释放失败层权重,支持随后重试。
TRELLIS 通过 empty-weights 初始化和 assign 加载权重,避免保留初始化权重的重复副本。

语音、LLM 及尚无可运行节点的 Motion / StdGEN 不在本次适配范围内。
首次反序列化、精度转换、LoRA 更新和写入映射文件仍可能出现主存峰值;
预检基于权重大小和预留空间,不代表能预测所有运行时激活开销。
超大单层、框架绕过模块调用、自定义量化 Tensor 或缺少原生卸载能力可能无法安全卸载,
会报告错误;不会偷偷降低精度、分辨率或帧数,也不会绕过授权尝试磁盘卸载。
mmap 还需要运行内存和 OS 页缓存,不能保证所有模型均可在任意内存大小运行。

## 验证边界

`yarn test_on tests.runtime_offload_test` 检查真实 CPU Tensor 的钩子生命周期、
磁盘映射输出一致性、失败恢复、LoRA 风格更新,以及实际适配器源代码的策略传递。
这些检查不需要下载巨型模型,不代表真实视频或 3D 模型的 GPU 推理、导出或质量验收。

接口:GET /api/resources/;POST/GET /api/resources/disk-requests;
GET/POST /api/resources/disk-requests/{id}。确认记录保留 180 秒,最多 128 条。
5 changes: 3 additions & 2 deletions extensions/3D-Model/ssui_3dmodel/Trellis.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,10 @@ def load(model_path: str, quantization: str = "none", dinov2_repository: str = N
if quantization != "none":
for name in ("sparse_structure_flow_model", "slat_flow_model", "image_cond_model"):
quantize(model.models[name])
if cpu_offload:
from backend.runtime_offload import LoadPlan
if cpu_offload or any(isinstance(getattr(m, '_ssui_load_plan', None), LoadPlan) for m in model.models.values()):
model._execution_device = torch.device("cuda")
ModelMemoryOptions(cpu_offload=True).configure(model.models.values(), device="cuda")
ModelMemoryOptions(cpu_offload=cpu_offload).configure(model.models.values(), device="cuda")
else:
model.cuda()
return TrellisModel(model_path, model)
Expand Down
Loading
Loading