From e1dc9ee99081d7e0a72118f8ef5ee7478f22b0da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A5=BF=E9=A3=8E=E9=80=8D=E9=81=A5=E6=B8=B8?= Date: Sat, 7 Nov 2026 00:31:44 +0800 Subject: [PATCH 1/2] feat: monitor memory and gate automatic model offload --- backend/disk_offload.py | 72 ++++++ .../load/model_cache/model_cache.py | 20 +- backend/resource_monitor.py | 34 +++ backend/runtime_offload.py | 176 +++++++++++++ doc/ResourceMonitoring.zh.md | 57 +++++ extensions/3D-Model/ssui_3dmodel/Trellis.py | 5 +- .../vendor/trellis/models/__init__.py | 3 + .../3D-Model/vendor/trellis/pipelines/base.py | 7 +- .../trellis/pipelines/trellis_image_to_3d.py | 4 + extensions/Image/ssui_image/api/model.py | 4 +- extensions/Video/README.md | 10 + extensions/Video/diffsynth/models/cog_vae.py | 4 +- .../models/hunyuan_video_vae_decoder.py | 2 +- .../models/hunyuan_video_vae_encoder.py | 2 +- .../Video/diffsynth/models/model_manager.py | 17 +- extensions/Video/diffsynth/pipelines/base.py | 4 + .../Video/diffsynth/pipelines/cog_video.py | 4 + .../Video/diffsynth/pipelines/wan_video.py | 6 + .../diffsynth/pipelines/wan_video_new.py | 6 + extensions/Video/diffsynth/utils/__init__.py | 14 +- extensions/Video/ssui_video/LTX2.py | 10 + extensions/Video/ssui_video/MiniMaxH3.py | 5 + extensions/Video/ssui_video/Wan22.py | 12 +- extensions/Video/ssui_video/h3_runner.py | 42 +++- frontend/functional_ui/src/App.tsx | 3 +- .../src/ui/common/ResourceMonitor.tsx | 52 ++++ frontend/ssui_components/src/api/generated.ts | 204 +++++++++++++++ server/app.py | 2 + server/contract_app.py | 2 + server/openapi.json | 181 +++++++++++++ server/routes/resource_routes.py | 67 +++++ ssui/model_memory.py | 20 +- tests/resource_monitor_test.py | 121 +++++++++ tests/runtime_offload_test.py | 238 ++++++++++++++++++ 34 files changed, 1381 insertions(+), 29 deletions(-) create mode 100644 backend/disk_offload.py create mode 100644 backend/resource_monitor.py create mode 100644 backend/runtime_offload.py create mode 100644 doc/ResourceMonitoring.zh.md create mode 100644 frontend/functional_ui/src/ui/common/ResourceMonitor.tsx create mode 100644 server/routes/resource_routes.py create mode 100644 tests/resource_monitor_test.py create mode 100644 tests/runtime_offload_test.py diff --git a/backend/disk_offload.py b/backend/disk_offload.py new file mode 100644 index 00000000..4d6f3ae4 --- /dev/null +++ b/backend/disk_offload.py @@ -0,0 +1,72 @@ +"""Opt-in disk backing for the existing partial-loading cache.""" +import json +import os +from pathlib import Path +import shutil +import sys +import tempfile +import time +from urllib.request import Request, urlopen + + +def confirm_disk_offload(model, required_bytes, timeout=180): + base = os.environ.get("SSUI_SERVER_URL", "http://localhost:7422").rstrip("/") + + def call(path, data=None): + body = None if data is None else json.dumps(data).encode() + request = Request(base + "/api/resources/" + path, data=body, + headers={"Content-Type": "application/json"}) + with urlopen(request, timeout=5) as response: + return json.load(response) + + try: + key = call("disk-requests", {"model": str(model), "required_bytes": required_bytes})["id"] + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + answer = call("disk-requests/" + key)["approved"] + if answer is not None: + if answer: + return + raise MemoryError("用户拒绝磁盘 offload,已停止模型加载。") + time.sleep(.5) + except MemoryError: + raise + except Exception as exc: + raise MemoryError("内存不足,无法取得磁盘 offload 授权;已停止加载。") from exc + raise MemoryError("磁盘 offload 确认超时,已停止加载。") + + +def disk_back_model(model, required_bytes): + """Own the backing directory for as long as the cache wrapper owns its tensors. + + mmap uses OS paging; it does not eliminate activation/working memory requirements. + Only the state dict written by this process is loaded, with weights_only=True. + """ + import torch + directory = new_offload_directory(required_bytes) + try: + path = Path(directory.name) / "weights.pt" + state = model.state_dict() + if any(t.device.type != "cpu" for t in state.values()): + raise RuntimeError("Disk backing requires a CPU model before device loading") + torch.save(state, path) + mapped = torch.load(path, mmap=True, weights_only=True, map_location="cpu") + model.load_state_dict(mapped, assign=True) + return directory + except Exception: + directory.cleanup() + raise + + +def new_offload_directory(required_bytes): + if sys.platform == "win32": + cache = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) + elif sys.platform == "darwin": + cache = Path.home() / "Library" / "Caches" + else: + cache = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) + root = Path(os.environ.get("SSUI_OFFLOAD_DIR", cache / "SSUI" / "offload")) + root.mkdir(parents=True, exist_ok=True) + if shutil.disk_usage(root).free < required_bytes + 2**30: + raise MemoryError("磁盘可用空间不足,无法 offload(需额外预留 1 GiB)。") + return tempfile.TemporaryDirectory(prefix="ssui-offload-", dir=root, ignore_cleanup_errors=True) diff --git a/backend/model_manager/load/model_cache/model_cache.py b/backend/model_manager/load/model_cache/model_cache.py index 83fa621b..515a9ac4 100644 --- a/backend/model_manager/load/model_cache/model_cache.py +++ b/backend/model_manager/load/model_cache/model_cache.py @@ -180,16 +180,32 @@ def put(self, key: str, model: AnyModel) -> None: # well. running_with_cuda = self._execution_device.type == "cuda" + backing = None + keep_ram_copy = self._keep_ram_copy_of_weights + if isinstance(model, torch.nn.Module) and running_with_cuda and self._enable_partial_loading: + from backend.resource_monitor import snapshot, placement + from backend.disk_offload import confirm_disk_offload, disk_back_model + resources = snapshot() + # The model is already resident: add its bytes back when judging host capacity. + resources["ram"]["available"] += size + mode = placement(size, resources, self._execution_device.index or 0, + int(self._execution_device_working_mem_gb * GB)) + if mode == "disk_confirmation": + confirm_disk_offload(key, size) + backing = disk_back_model(model, size) + keep_ram_copy = True + # Wrap model. if isinstance(model, torch.nn.Module) and running_with_cuda and self._enable_partial_loading: wrapped_model = CachedModelWithPartialLoad( - model, self._execution_device, keep_ram_copy=self._keep_ram_copy_of_weights + model, self._execution_device, keep_ram_copy=keep_ram_copy ) else: wrapped_model = CachedModelOnlyFullLoad( - model, self._execution_device, size, keep_ram_copy=self._keep_ram_copy_of_weights + model, self._execution_device, size, keep_ram_copy=keep_ram_copy ) + wrapped_model._disk_backing = backing cache_record = CacheRecord(key=key, cached_model=wrapped_model) self._cached_models[key] = cache_record self._cache_stack.append(key) diff --git a/backend/resource_monitor.py b/backend/resource_monitor.py new file mode 100644 index 00000000..5d5c941b --- /dev/null +++ b/backend/resource_monitor.py @@ -0,0 +1,34 @@ +"""Machine resources and conservative model placement decisions (all sizes are bytes).""" +import psutil + +GiB = 2**30 + + +def snapshot(): + ram = psutil.virtual_memory() + result = {"ram": {"total": ram.total, "available": ram.available, + "used": ram.total - ram.available}, "gpus": [], "gpu_error": None} + try: + import torch + if torch.cuda.is_available(): + for index in range(torch.cuda.device_count()): + free, total = torch.cuda.mem_get_info(index) + result["gpus"].append({"index": index, "name": torch.cuda.get_device_name(index), + "total": total, "available": free, "used": total - free}) + except Exception as exc: + result["gpu_error"] = str(exc) + return result + + +def placement(weight_bytes, resources, device_index=0, working_bytes=3 * GiB): + """Reserve runtime headroom; unknown VRAM is never treated as unlimited.""" + if weight_bytes < 0 or working_bytes < 0: + raise ValueError("Memory estimates must be non-negative") + reserve = max(GiB, int(resources["ram"]["total"] * .1)) + # This cache retains a CPU copy even when all weights fit on the GPU. + if weight_bytes + reserve > resources["ram"]["available"]: + return "disk_confirmation" + gpu = next((g for g in resources["gpus"] if g["index"] == device_index), None) + if gpu and weight_bytes + working_bytes <= gpu["available"]: + return "device" + return "cpu_offload" diff --git a/backend/runtime_offload.py b/backend/runtime_offload.py new file mode 100644 index 00000000..b539e733 --- /dev/null +++ b/backend/runtime_offload.py @@ -0,0 +1,176 @@ +"""Shared placement policy for extension model loaders and independent runners.""" +from dataclasses import dataclass +from functools import wraps +from pathlib import Path +import torch + +from backend.resource_monitor import GiB, placement, snapshot +from backend.disk_offload import confirm_disk_offload, disk_back_model + +_WEIGHT_SUFFIXES = {'.safetensors', '.pt', '.pth', '.bin', '.ckpt'} + + +def weight_size(paths): + """Conservative on-disk estimate; deduplicate shards and exclude tokenizer/config files.""" + if isinstance(paths, (str, Path)): + paths = [paths] + files = set() + for value in paths: + path = Path(value) + if path.is_dir(): + files.update(p.resolve() for p in path.rglob('*') if p.suffix in _WEIGHT_SUFFIXES and p.is_file()) + elif path.is_file() and path.suffix in _WEIGHT_SUFFIXES: + files.add(path.resolve()) + return sum(p.stat().st_size for p in files) + + +@dataclass +class LoadPlan: + mode: str + device: str + estimated_bytes: int + + +def plan_load(label, paths=(), device='cuda', requested=None, estimated_bytes=None): + """Ask before CPU staging starts. A request authorizes only this load invocation.""" + size = weight_size(paths) if estimated_bytes is None else estimated_bytes + if size <= 0: + raise MemoryError(f'{label}: 无法估算模型权重大小,已停止自动加载。') + target = torch.device(device) + resources = snapshot() + index = target.index if target.index is not None else (torch.cuda.current_device() if target.type == 'cuda' and torch.cuda.is_available() else 0) + mode = placement(size, resources, device_index=index) + if requested == 'disk' or mode == 'disk_confirmation': + confirm_disk_offload(label, size) + mode = 'disk_offload' + elif requested == 'cpu' or target.type != 'cuda': + mode = 'cpu_offload' + return LoadPlan(mode, str(target), size) + + +def configure_model(model, plan): + """Use Accelerate layer hooks; keep the execution device separate from meta weights. + + Disk mode supplies mmap-backed CPU tensors to the same hooks. Offload weights are + never moved en masse by pipeline stage transitions. Custom tensor subclasses are + rejected before hooks are installed: silently dequantizing them is unsafe. + """ + if getattr(model, '_ssui_residency', None) is not None: + return model + if plan.mode == 'device': + model.to(plan.device) + return model + if any(type(p) is not torch.nn.Parameter for p in model.parameters()): + raise MemoryError('该模型的自定义量化权重不支持自动 offload,请使用框架原生 offload。') + from accelerate import cpu_offload + model.cpu() + backing = None + try: + size = sum(t.numel() * t.element_size() for t in model.state_dict().values()) + if plan.mode == 'disk_offload': + backing = disk_back_model(model, size) + # Some attention implementations read a child's weights directly. + preload = ['MultiheadAttention'] + cpu_offload(model, execution_device=torch.device(plan.device), offload_buffers=False, + state_dict=model.state_dict(), preload_module_classes=preload) + # Accelerate's forward wrapper does not run post_forward after an exception. + # PyTorch always_call hooks release the failing layer and its ancestors. + handles = [] + def release_on_failure(module, args, output): + if output is None and hasattr(module, '_hf_hook'): + module._hf_hook.post_forward(module, None) + for module in model.modules(): + if hasattr(module, '_hf_hook'): + handles.append(module.register_forward_hook(release_on_failure, always_call=True)) + model._ssui_failure_hooks = handles + model._ssui_plan = plan + model._ssui_execution_device = torch.device(plan.device) + model._ssui_residency = plan.mode + model._ssui_disk_backing = backing + except Exception: + # Remove partially installed hooks and restore CPU weights before cleaning files. + from accelerate.hooks import remove_hook_from_module + remove_hook_from_module(model, recurse=True) + if backing is not None: + backing.cleanup() + raise + return model + + +def is_managed(model): + return getattr(model, '_ssui_residency', None) in ('cpu_offload', 'disk_offload') + + +def remote_weight_size(repo_id, prefixes=()): + """Read Hub file metadata without downloading the checkpoint to RAM.""" + local = Path(repo_id) + if local.exists(): + return weight_size(local) + from huggingface_hub import HfApi + info = HfApi().model_info(repo_id, files_metadata=True) + files = [f for f in info.siblings if Path(f.rfilename).suffix in _WEIGHT_SUFFIXES + and (not prefixes or any(f.rfilename.startswith(p + '/') for p in prefixes))] + if not files or any(f.size is None for f in files): + raise MemoryError(f'{repo_id}: 无法读取权重大小,已停止自动加载。') + return sum(f.size for f in files) + + +def materialize_for_update(method): + """Temporarily restore managed weights for existing DiffSynth LoRA editors.""" + @wraps(method) + def wrapped(self, *args, **kwargs): + from accelerate.hooks import remove_hook_from_module + managed = [(m, m._ssui_plan, getattr(m, '_ssui_disk_backing', None)) + for m in self.model if is_managed(m) and hasattr(m, "_ssui_plan")] + for model, plan, backing in managed: + for handle in model._ssui_failure_hooks: + handle.remove() + remove_hook_from_module(model, recurse=True) + model._ssui_residency = None + try: + return method(self, *args, **kwargs) + finally: + for model, plan, backing in managed: + configure_model(model, plan) + if backing is not None: + backing.cleanup() + return wrapped + + +def configure_pending(model, device, force_cpu=False): + """Finalize after LoRA/quantization; checkpoint loaders only stage on CPU.""" + pending = getattr(model, '_ssui_load_plan', None) + if pending is None or is_managed(model): + return is_managed(model) + from dataclasses import replace + plan = replace(pending, device=str(device)) + if force_cpu and plan.mode == 'device': + plan = replace(plan, mode='cpu_offload') + if any('float8' in str(p.dtype) for p in model.parameters()): + if plan.mode == 'disk_offload': + raise MemoryError('FP8 存储使用框架原生精度转换;自动磁盘卸载请改用 none/int8。') + del model._ssui_load_plan + return False + if getattr(model, '_ssui_int8', False): + # Native packed Linear stores weights in buffers and exposes a computed + # weight property. Keep its native method/generator residency wrapper. + from ssui.model_memory import offload_component + model.cpu() + if plan.mode != 'device': + backing = disk_back_model(model, sum(t.numel()*t.element_size() for t in model.state_dict().values())) if plan.mode == 'disk_offload' else None + offload_component(model, device, cpu_state=model.state_dict() if backing else None) + model._ssui_disk_backing = backing + model._ssui_residency = plan.mode + model._ssui_execution_device = torch.device(device) + else: + model.to(device) + else: + configure_model(model, plan) + del model._ssui_load_plan + return is_managed(model) + + +def finalize_pipeline(pipe): + for module in pipe.children(): + configure_pending(module, pipe.device) + return any(is_managed(m) for m in pipe.children()) diff --git a/doc/ResourceMonitoring.zh.md b/doc/ResourceMonitoring.zh.md new file mode 100644 index 00000000..93ef54eb --- /dev/null +++ b/doc/ResourceMonitoring.zh.md @@ -0,0 +1,57 @@ +# 内存、显存和模型 offload + +功能 UI 顶部每 2 秒显示服务端机器的系统内存与各 CUDA 显卡的已用 / 总量。 +内存按 total - available 计算;CUDA 显存按驱动 free / total 计算,包含其他进程占用。 +未检测到 CUDA 与读取失败分别显示。Apple 共享内存设备当前只显示系统内存,不提供独立显存读数。 +远程服务时显示服务端机器,非浏览器机器。 + +图片模型的 ModelLoaderService 和 load_model_from_path 已启用现有 CUDA 部分加载: +缓存会清理未锁定模型,保留默认 3 GiB 执行空间,按剩余显存加载权重,其余留在 CPU, +由现有自定义层按需转移。保留空间是估算,不保证任何分辨率和批次都不会 OOM。 + +进入缓存时发现主存余量不足(预留至少 1 GiB 或总内存 10%),会发起一次性磁盘确认。 +允许后将 CPU 权重写入临时文件并 mmap 映射,仍使用同一部分加载与卸载路径。 +拒绝、180 秒未答复、服务端不可达或磁盘空间不足都会停止加载。 +磁盘至少需要权重大小加 1 GiB 可用空间;模型缓存释放后临时目录随对象回收。 +进程异常退出可能留下 ssui-offload-* 临时目录,可在关闭应用后清理。 + +执行器默认连接 http://localhost:7422,通过 SSUI_SERVER_URL 指定其他服务端地址。 +SSUI_OFFLOAD_DIR 可指定磁盘缓存根目录;默认为用户缓存目录下的 SSUI/offload。 +请把它放在有足够空间的真实磁盘上:Linux /tmp 可能是 tmpfs,无法缓解主存不足。 +授权仅适用于本次请求,不会记住为永久许可。 + +## 当前覆盖范围 + +| 入口 | 自动 CPU offload | 经确认的磁盘 offload | +| --- | --- | --- | +| 图片 ModelCache | 现有 CUDA 部分加载 | CPU 权重 mmap | +| 视频 Wan / Wan2 / Wan2.2、CogVideo、Hunyuan、AnimateDiff | DiffSynth 加载前预算 + Accelerate 逐层钩子 | mmap CPU 权重 + 同一逐层钩子 | +| TRELLIS 3D | 每个组件加载前预算 + 逐层钩子,执行设备独立保存 | 同一授权与 mmap 路径 | +| LTX 独立进程 | 主进程预检后设置原生 --offload cpu | 原生 --offload disk,仅授权后传入 | +| H3 独立进程 | Hub 文件大小预检 + ComponentsManager CPU offload / int8 组卸载 | 检查 diffusers 原生磁盘组卸载能力后启用 | + +视频按整个模型组估算后再加载首个组件;TRELLIS 逐个组件检查实时余量。 +本地权重文件统计会去重,目录仅统计权重文件;仓库未提供可靠大小时停止自动加载。 +H3 使用 Hub 文件大小元数据,不会为估算预先把权重读进 RAM。 +逐层钩子与原有阶段 .to/.cpu、Wan 显存管理保持互斥;VAE 的执行设备不从 meta 权重推断。 +加载器先在 CPU 上完成 LoRA / INT8 转换,再安装卸载策略;原生 INT8 保留组件方法和生成器的驻留包装。 +FP8 保留框架原生存储/计算精度转换;当前自动磁盘路径对 FP8 明确拒绝,请使用 none/int8。 +DiffSynth 的 LoRA 更新会暂时恢复 CPU 权重,更新后重新安装同一卸载模式;更新仍可能产生内存峰值。 +推理异常会释放失败层权重,支持随后重试。 +TRELLIS 通过 empty-weights 初始化和 assign 加载权重,避免保留初始化权重的重复副本。 + +语音、LLM 及尚无可运行节点的 Motion / StdGEN 不在本次适配范围内。 +首次反序列化、精度转换、LoRA 更新和写入映射文件仍可能出现主存峰值; +预检基于权重大小和预留空间,不代表能预测所有运行时激活开销。 +超大单层、框架绕过模块调用、自定义量化 Tensor 或缺少原生卸载能力可能无法安全卸载, +会报告错误;不会偷偷降低精度、分辨率或帧数,也不会绕过授权尝试磁盘卸载。 +mmap 还需要运行内存和 OS 页缓存,不能保证所有模型均可在任意内存大小运行。 + +## 验证边界 + +`yarn test_on tests.runtime_offload_test` 检查真实 CPU Tensor 的钩子生命周期、 +磁盘映射输出一致性、失败恢复、LoRA 风格更新,以及实际适配器源代码的策略传递。 +这些检查不需要下载巨型模型,不代表真实视频或 3D 模型的 GPU 推理、导出或质量验收。 + +接口:GET /api/resources/;POST/GET /api/resources/disk-requests; +GET/POST /api/resources/disk-requests/{id}。确认记录保留 180 秒,最多 128 条。 diff --git a/extensions/3D-Model/ssui_3dmodel/Trellis.py b/extensions/3D-Model/ssui_3dmodel/Trellis.py index 545a77c8..255ac551 100644 --- a/extensions/3D-Model/ssui_3dmodel/Trellis.py +++ b/extensions/3D-Model/ssui_3dmodel/Trellis.py @@ -53,9 +53,10 @@ def load(model_path: str, quantization: str = "none", dinov2_repository: str = N if quantization != "none": for name in ("sparse_structure_flow_model", "slat_flow_model", "image_cond_model"): quantize(model.models[name]) - if cpu_offload: + from backend.runtime_offload import LoadPlan + if cpu_offload or any(isinstance(getattr(m, '_ssui_load_plan', None), LoadPlan) for m in model.models.values()): model._execution_device = torch.device("cuda") - ModelMemoryOptions(cpu_offload=True).configure(model.models.values(), device="cuda") + ModelMemoryOptions(cpu_offload=cpu_offload).configure(model.models.values(), device="cuda") else: model.cuda() return TrellisModel(model_path, model) diff --git a/extensions/3D-Model/vendor/trellis/models/__init__.py b/extensions/3D-Model/vendor/trellis/models/__init__.py index 8973161a..d444b2c0 100644 --- a/extensions/3D-Model/vendor/trellis/models/__init__.py +++ b/extensions/3D-Model/vendor/trellis/models/__init__.py @@ -59,9 +59,12 @@ def from_pretrained(path: str, **kwargs): # Avoid allocating a second full parameter set on 16 GB hosts. Safetensors # maps checkpoint pages and assign=True preserves those tensors directly. from accelerate import init_empty_weights + from backend.runtime_offload import plan_load + plan = plan_load(f"TRELLIS {path}", model_file) with init_empty_weights(): model = __getattr__(config['name'])(**config['args'], **kwargs) model.load_state_dict(load_file(model_file), assign=True) + model._ssui_load_plan = plan return model diff --git a/extensions/3D-Model/vendor/trellis/pipelines/base.py b/extensions/3D-Model/vendor/trellis/pipelines/base.py index 4d7c770c..a052988d 100644 --- a/extensions/3D-Model/vendor/trellis/pipelines/base.py +++ b/extensions/3D-Model/vendor/trellis/pipelines/base.py @@ -58,8 +58,13 @@ def device(self) -> torch.device: raise RuntimeError("No device found.") def to(self, device: torch.device) -> None: + from backend.runtime_offload import is_managed + self._execution_device = torch.device(device) for model in self.models.values(): - model.to(device) + if not is_managed(model): + model.to(device) + elif torch.device(device) != model._ssui_execution_device: + raise RuntimeError("Reload managed TRELLIS models to change execution device") def cuda(self) -> None: self.to(torch.device("cuda")) diff --git a/extensions/3D-Model/vendor/trellis/pipelines/trellis_image_to_3d.py b/extensions/3D-Model/vendor/trellis/pipelines/trellis_image_to_3d.py index 9cc795b6..d3d9f62d 100644 --- a/extensions/3D-Model/vendor/trellis/pipelines/trellis_image_to_3d.py +++ b/extensions/3D-Model/vendor/trellis/pipelines/trellis_image_to_3d.py @@ -88,6 +88,10 @@ def _init_image_cond_model(self, name: str, local_repo: str = None, local_weight dinov2_model = torch.hub.load(source, name, source=source_kind, pretrained=True) dinov2_model.eval() + from backend.runtime_offload import plan_load + size = sum(t.numel() * t.element_size() for t in dinov2_model.state_dict().values()) + plan = plan_load(f"TRELLIS {name}", estimated_bytes=size) + dinov2_model._ssui_load_plan = plan self.models['image_cond_model'] = dinov2_model transform = transforms.Compose([ transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), diff --git a/extensions/Image/ssui_image/api/model.py b/extensions/Image/ssui_image/api/model.py index dd47b03a..b74ab02d 100644 --- a/extensions/Image/ssui_image/api/model.py +++ b/extensions/Image/ssui_image/api/model.py @@ -81,7 +81,7 @@ def load_model_from_path( ) -> LoadedModelWithoutConfig: ram_cache = ModelCache( execution_device_working_mem_gb=3, - enable_partial_loading=False, + enable_partial_loading=True, keep_ram_copy_of_weights=True, max_ram_cache_size_gb=None, max_vram_cache_size_gb=None, @@ -134,7 +134,7 @@ def __init__(self, *, quantization: str = "none", cpu_offload: bool = False): self._app_config = ModelLoaderConfig() self._ram_cache = ModelCache( execution_device_working_mem_gb=3, - enable_partial_loading=False, + enable_partial_loading=True, keep_ram_copy_of_weights=True, max_ram_cache_size_gb=None, max_vram_cache_size_gb=None, diff --git a/extensions/Video/README.md b/extensions/Video/README.md index 91b9fafb..25b0862f 100644 --- a/extensions/Video/README.md +++ b/extensions/Video/README.md @@ -463,3 +463,13 @@ SSUI_VENV_PATH="$PWD/.venv/video-mps" yarn test:ltx:smoke 默认 128×128、9 帧;runner 禁止 CPU 回退,测试记录实际层输出设备,并由父进程 监控系统可用内存及 swap 增长。该保护不是系统绝不会卡顿的保证。 + +### 自动资源策略 + +视频与 TRELLIS 3D 入口共用 SSUI 的内存、显存预算和一次性磁盘确认。 +DiffSynth 的 Wan / Wan2 / Wan2.2、CogVideo、Hunyuan、AnimateDiff 在 CPU 上加载, +再按预算常驻显卡或使用逐层卸载;不会同时套用两套显存管理钩子。 +LTX 会根据预算调整原生 `--offload`,即使手动选择 disk 也必须通过一次性确认。 +H3 独立环境需安装 psutil,使用 Hub 权重大小元数据做预检;旧版 diffusers 缺少 +`offload_to_disk_path` 时会明确停止,请升级独立环境。 +资源策略和验证边界见 [资源监控说明](../../doc/ResourceMonitoring.zh.md)。 diff --git a/extensions/Video/diffsynth/models/cog_vae.py b/extensions/Video/diffsynth/models/cog_vae.py index 24ab3b3f..ebabfdff 100644 --- a/extensions/Video/diffsynth/models/cog_vae.py +++ b/extensions/Video/diffsynth/models/cog_vae.py @@ -253,7 +253,7 @@ def decode_video(self, sample, tiled=True, tile_size=(60, 90), tile_stride=(30, def decode_small_video(self, sample): B, C, T, H, W = sample.shape - computation_device = self.conv_in.weight.device + computation_device = getattr(self, "_ssui_execution_device", self.conv_in.weight.device) computation_dtype = self.conv_in.weight.dtype value = [] for i in range(T//2): @@ -338,7 +338,7 @@ def encode_video(self, sample, tiled=True, tile_size=(60, 90), tile_stride=(30, def encode_small_video(self, sample): B, C, T, H, W = sample.shape - computation_device = self.conv_in.weight.device + computation_device = getattr(self, "_ssui_execution_device", self.conv_in.weight.device) computation_dtype = self.conv_in.weight.dtype value = [] for i in range(T//8): diff --git a/extensions/Video/diffsynth/models/hunyuan_video_vae_decoder.py b/extensions/Video/diffsynth/models/hunyuan_video_vae_decoder.py index ae09ff85..64e9fab4 100644 --- a/extensions/Video/diffsynth/models/hunyuan_video_vae_decoder.py +++ b/extensions/Video/diffsynth/models/hunyuan_video_vae_decoder.py @@ -448,7 +448,7 @@ def tile_forward(self, hidden_states, tile_size, tile_stride): # Run torch_dtype = self.post_quant_conv.weight.dtype data_device = hidden_states.device - computation_device = self.post_quant_conv.weight.device + computation_device = getattr(self, "_ssui_execution_device", self.post_quant_conv.weight.device) weight = torch.zeros((1, 1, (T - 1) * 4 + 1, H * 8, W * 8), dtype=torch_dtype, device=data_device) values = torch.zeros((B, 3, (T - 1) * 4 + 1, H * 8, W * 8), dtype=torch_dtype, device=data_device) diff --git a/extensions/Video/diffsynth/models/hunyuan_video_vae_encoder.py b/extensions/Video/diffsynth/models/hunyuan_video_vae_encoder.py index faaaeb95..ed3195c9 100644 --- a/extensions/Video/diffsynth/models/hunyuan_video_vae_encoder.py +++ b/extensions/Video/diffsynth/models/hunyuan_video_vae_encoder.py @@ -247,7 +247,7 @@ def tile_forward(self, hidden_states, tile_size, tile_stride): # Run torch_dtype = self.quant_conv.weight.dtype data_device = hidden_states.device - computation_device = self.quant_conv.weight.device + computation_device = getattr(self, "_ssui_execution_device", self.quant_conv.weight.device) weight = torch.zeros((1, 1, (T - 1) // 4 + 1, H // 8, W // 8), dtype=torch_dtype, device=data_device) values = torch.zeros((B, 16, (T - 1) // 4 + 1, H // 8, W // 8), dtype=torch_dtype, device=data_device) diff --git a/extensions/Video/diffsynth/models/model_manager.py b/extensions/Video/diffsynth/models/model_manager.py index d46eedfa..6d2e3cff 100644 --- a/extensions/Video/diffsynth/models/model_manager.py +++ b/extensions/Video/diffsynth/models/model_manager.py @@ -1,5 +1,6 @@ import os, torch, json, importlib from typing import List +from backend.runtime_offload import materialize_for_update from .downloader import download_models, download_customized_models, Preset_model_id, Preset_model_website @@ -370,6 +371,7 @@ def load_patch_model_from_single_file(self, file_path="", state_dict={}, model_n print(f" The following patched models are loaded: {model_names}.") + @materialize_for_update def load_lora(self, file_path="", state_dict={}, lora_alpha=1.0): if isinstance(file_path, list): for file_path_ in file_path: @@ -392,10 +394,14 @@ def load_lora(self, file_path="", state_dict={}, lora_alpha=1.0): print(f" Cannot load LoRA: {file_path}") - def load_model(self, file_path, model_names=None, device=None, torch_dtype=None): + def load_model(self, file_path, model_names=None, device=None, torch_dtype=None, resource_plan=None): print(f"Loading models from: {file_path}") if device is None: device = self.device if torch_dtype is None: torch_dtype = self.torch_dtype + from backend.runtime_offload import plan_load, configure_model + # self.device is the execution device, even when callers request CPU staging. + resource_plan = resource_plan or plan_load(str(file_path), file_path, self.device) + device = "cpu" if isinstance(file_path, list): state_dict = {} for path in file_path: @@ -412,6 +418,7 @@ def load_model(self, file_path, model_names=None, device=None, torch_dtype=None) allowed_model_names=model_names, model_manager=self ) for model_name, model in zip(model_names, models): + model._ssui_load_plan = resource_plan self.model.append(model) self.model_path.append(file_path) self.model_name.append(model_name) @@ -422,8 +429,11 @@ def load_model(self, file_path, model_names=None, device=None, torch_dtype=None) def load_models(self, file_path_list, model_names=None, device=None, torch_dtype=None): + from backend.runtime_offload import plan_load + # Budget the complete group before loading its first component. + plan = plan_load("视频模型组", [p for item in file_path_list for p in (item if isinstance(item, list) else [item])], self.device) for file_path in file_path_list: - self.load_model(file_path, model_names, device=device, torch_dtype=torch_dtype) + self.load_model(file_path, model_names, device=device, torch_dtype=torch_dtype, resource_plan=plan) def fetch_model(self, model_name, file_path=None, require_model_path=False, index=None): @@ -463,5 +473,6 @@ def fetch_model(self, model_name, file_path=None, require_model_path=False, inde def to(self, device): for model in self.model: - model.to(device) + if not getattr(model, "_ssui_residency", None): + model.to(device) diff --git a/extensions/Video/diffsynth/pipelines/base.py b/extensions/Video/diffsynth/pipelines/base.py index 5b89fb49..614c304c 100644 --- a/extensions/Video/diffsynth/pipelines/base.py +++ b/extensions/Video/diffsynth/pipelines/base.py @@ -102,6 +102,8 @@ def load_models_to_device(self, loadmodel_names=[]): if model_name not in loadmodel_names: model = getattr(self, model_name) if model is not None: + if getattr(model, "_ssui_residency", None): + continue if hasattr(model, "vram_management_enabled") and model.vram_management_enabled: for module in model.modules(): if hasattr(module, "offload"): @@ -112,6 +114,8 @@ def load_models_to_device(self, loadmodel_names=[]): for model_name in loadmodel_names: model = getattr(self, model_name) if model is not None: + if getattr(model, "_ssui_residency", None): + continue if hasattr(model, "vram_management_enabled") and model.vram_management_enabled: for module in model.modules(): if hasattr(module, "onload"): diff --git a/extensions/Video/diffsynth/pipelines/cog_video.py b/extensions/Video/diffsynth/pipelines/cog_video.py index fee3030e..f65513e6 100644 --- a/extensions/Video/diffsynth/pipelines/cog_video.py +++ b/extensions/Video/diffsynth/pipelines/cog_video.py @@ -25,6 +25,10 @@ def __init__(self, device="cuda", torch_dtype=torch.float16): self.model_names = ["text_encoder", "dit", "vae_encoder", "vae_decoder"] def enable_vram_management(self): + from backend.runtime_offload import finalize_pipeline + if finalize_pipeline(self): + self.cpu_offload = True + return # Keep DiT weights on the host; transfer only the current layer to MPS. self.enable_cpu_offload() self.dit.computation_device = self.device diff --git a/extensions/Video/diffsynth/pipelines/wan_video.py b/extensions/Video/diffsynth/pipelines/wan_video.py index 2b5126a5..a16da20b 100644 --- a/extensions/Video/diffsynth/pipelines/wan_video.py +++ b/extensions/Video/diffsynth/pipelines/wan_video.py @@ -43,6 +43,12 @@ def __init__(self, device="cuda", torch_dtype=torch.float16, tokenizer_path=None def enable_vram_management(self, num_persistent_param_in_dit=None): + from backend.runtime_offload import finalize_pipeline + finalize_pipeline(self) + if any(getattr(m, "_ssui_residency", None) for m in self.modules()): + self.cpu_offload = True + self.vram_management_enabled = True + return dtype = next(iter(self.text_encoder.parameters())).dtype enable_vram_management( self.text_encoder, diff --git a/extensions/Video/diffsynth/pipelines/wan_video_new.py b/extensions/Video/diffsynth/pipelines/wan_video_new.py index 2fc52977..32cf83a6 100644 --- a/extensions/Video/diffsynth/pipelines/wan_video_new.py +++ b/extensions/Video/diffsynth/pipelines/wan_video_new.py @@ -128,6 +128,12 @@ def training_loss(self, **inputs): def enable_vram_management(self, num_persistent_param_in_dit=None, vram_limit=None, vram_buffer=0.5): + from backend.runtime_offload import finalize_pipeline + finalize_pipeline(self) + if any(getattr(m, "_ssui_residency", None) for m in self.modules()): + self.cpu_offload = True + self.vram_management_enabled = True + return self.vram_management_enabled = True if num_persistent_param_in_dit is not None: vram_limit = None diff --git a/extensions/Video/diffsynth/utils/__init__.py b/extensions/Video/diffsynth/utils/__init__.py index 033a8f20..f8c99bd9 100644 --- a/extensions/Video/diffsynth/utils/__init__.py +++ b/extensions/Video/diffsynth/utils/__init__.py @@ -38,7 +38,15 @@ def to(self, *args, **kwargs): self.device = device if dtype is not None: self.torch_dtype = dtype - super().to(*args, **kwargs) + from backend.runtime_offload import finalize_pipeline, is_managed + finalize_pipeline(self) + if any(is_managed(m) for m in self.children()): + for child in self.children(): + if not is_managed(child): + child.to(*args, **kwargs) + self.vram_management_enabled = True + else: + super().to(*args, **kwargs) return self @@ -97,6 +105,8 @@ def load_models_to_device(self, model_names=[]): if self.vram_management_enabled: # offload models for name, model in self.named_children(): + if getattr(model, "_ssui_residency", None): + continue if name not in model_names: if hasattr(model, "vram_management_enabled") and model.vram_management_enabled: for module in model.modules(): @@ -107,6 +117,8 @@ def load_models_to_device(self, model_names=[]): empty_device_cache(self.device) # onload models for name, model in self.named_children(): + if getattr(model, "_ssui_residency", None): + continue if name in model_names: if hasattr(model, "vram_management_enabled") and model.vram_management_enabled: for module in model.modules(): diff --git a/extensions/Video/ssui_video/LTX2.py b/extensions/Video/ssui_video/LTX2.py index 81d92930..80135f08 100644 --- a/extensions/Video/ssui_video/LTX2.py +++ b/extensions/Video/ssui_video/LTX2.py @@ -71,6 +71,16 @@ def _ltx_python() -> str: def _run_ltx(module: str, args: list[str]) -> None: """调用 LTX venv 的 CLI,实时转发输出,失败时抛出带日志的错误。""" + from backend.runtime_offload import plan_load + args = list(args) + paths = [args[i + 1] for i, value in enumerate(args[:-1]) if value.endswith("-path") and value != "--output-path"] + requested = args[args.index("--offload") + 1] if "--offload" in args else None + plan = plan_load("LTX-2.5", paths, requested=requested) + if "--offload" in args: + index = args.index("--offload") + del args[index:index + 2] + if plan.mode != "device": + args += ["--offload", "disk" if plan.mode == "disk_offload" else "cpu"] python = _ltx_python() cmd = [python, os.path.join(os.path.dirname(__file__), "ltx_runner.py"), "--module", module, *args] proc = subprocess.Popen( diff --git a/extensions/Video/ssui_video/MiniMaxH3.py b/extensions/Video/ssui_video/MiniMaxH3.py index 7c9cf25a..60817125 100644 --- a/extensions/Video/ssui_video/MiniMaxH3.py +++ b/extensions/Video/ssui_video/MiniMaxH3.py @@ -103,8 +103,13 @@ def _run_h3( last_image._image.save(last_path) cmd += ["--last-image", last_path] + import backend + env = os.environ.copy() + root = os.path.dirname(os.path.dirname(backend.__file__)) + env["PYTHONPATH"] = root + os.pathsep + env.get("PYTHONPATH", "") proc = subprocess.Popen( cmd, + env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, diff --git a/extensions/Video/ssui_video/Wan22.py b/extensions/Video/ssui_video/Wan22.py index 496ad7ea..403ffa99 100644 --- a/extensions/Video/ssui_video/Wan22.py +++ b/extensions/Video/ssui_video/Wan22.py @@ -84,7 +84,9 @@ def load(quantization: str = "none", lora_path: str = "", lora_weight: float = 1 for name, component in pipe.named_children(): if name in ("dit", "dit2", "text_encoder", "image_encoder"): options.quantize(component) - component.to("cpu" if cpu_offload else runtime["device"]) + from backend.runtime_offload import configure_pending + if not configure_pending(component, runtime["device"], cpu_offload): + component.to("cpu" if cpu_offload else runtime["device"]) pipe.vram_management_enabled = cpu_offload elif cpu_offload: pipe.enable_vram_management(num_persistent_param_in_dit=None) @@ -157,7 +159,9 @@ def load( for name, component in pipe.named_children(): if name in ("dit", "dit2", "text_encoder", "image_encoder"): options.quantize(component) - component.to("cpu" if cpu_offload else runtime["device"]) + from backend.runtime_offload import configure_pending + if not configure_pending(component, runtime["device"], cpu_offload): + component.to("cpu" if cpu_offload else runtime["device"]) pipe.vram_management_enabled = cpu_offload elif cpu_offload: pipe.enable_vram_management(num_persistent_param_in_dit=None) @@ -230,7 +234,9 @@ def load( for name, component in pipe.named_children(): if name in ("dit", "dit2", "text_encoder", "image_encoder"): options.quantize(component) - component.to("cpu" if cpu_offload else runtime["device"]) + from backend.runtime_offload import configure_pending + if not configure_pending(component, runtime["device"], cpu_offload): + component.to("cpu" if cpu_offload else runtime["device"]) pipe.vram_management_enabled = cpu_offload elif cpu_offload: pipe.enable_vram_management(num_persistent_param_in_dit=None) diff --git a/extensions/Video/ssui_video/h3_runner.py b/extensions/Video/ssui_video/h3_runner.py index 3ba6e3ec..5dbeea19 100644 --- a/extensions/Video/ssui_video/h3_runner.py +++ b/extensions/Video/ssui_video/h3_runner.py @@ -16,6 +16,7 @@ import time import sys import traceback +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) def _guard_mps_memory(pipe): @@ -93,7 +94,7 @@ def _checkpoint_offload(model, checkpoint_dir): cpu_offload(model, execution_device=torch.device("mps"), offload_buffers=True, state_dict=weights) -def _enable_offload(pipe, device="cuda", model_root=None) -> None: +def _enable_offload(pipe, device="cuda", model_root=None, disk_directory=None) -> None: import torch from diffusers.hooks import apply_group_offloading @@ -111,12 +112,16 @@ def _enable_offload(pipe, device="cuda", model_root=None) -> None: offload = dict( onload_device=torch.device(device), offload_device=torch.device("cpu"), - use_stream=torch.device(device).type == "cuda", + use_stream=torch.device(device).type == "cuda" and disk_directory is None, ) + transformer_options, text_options = dict(offload), dict(offload) + if disk_directory is not None: + transformer_options["offload_to_disk_path"] = str(Path(disk_directory) / "transformer") + text_options["offload_to_disk_path"] = str(Path(disk_directory) / "text_encoder") pipe.transformer.enable_group_offload( - offload_type="block_level", num_blocks_per_group=1, **offload + offload_type="block_level", num_blocks_per_group=1, **transformer_options ) - apply_group_offloading(pipe.text_encoder.model, offload_type="leaf_level", **offload) + apply_group_offloading(pipe.text_encoder.model, offload_type="leaf_level", **text_options) pipe.vae.to(device) pipe.audio_vae.to(device) @@ -155,6 +160,7 @@ def main() -> int: ) return 1 + disk_directory = None try: device = args.device if device == "auto": @@ -166,8 +172,21 @@ def main() -> int: if device == "mps": torch.set_num_threads(4) torch.mps.set_per_process_memory_fraction(0.35) + from backend.runtime_offload import plan_load, remote_weight_size + from backend.disk_offload import new_offload_directory + transformer = "transformer" if args.task in ("t2va", "fl2va") else "transformer_ref" + size = remote_weight_size(args.model_id, (transformer, "text_encoder", "vae", "audio_vae")) + plan = plan_load("MiniMax H3", device=device, estimated_bytes=size, + requested="cpu" if args.cpu_offload else None) + args.cpu_offload = plan.mode != "device" + if plan.mode == "disk_offload" and device == "cuda": + import inspect + from diffusers.hooks import apply_group_offloading + if "offload_to_disk_path" not in inspect.signature(apply_group_offloading).parameters: + raise MemoryError("当前 H3 diffusers 不支持磁盘 offload,请升级 H3 独立环境。") + disk_directory = new_offload_directory(size) manager = ComponentsManager() - if device == "cuda" and args.cpu_offload and args.quantization == "none": + if device == "cuda" and args.cpu_offload and args.quantization == "none" and disk_directory is None: manager.enable_auto_cpu_offload(device=device) pipe = ModularPipeline.from_pretrained( args.model_id, workflow=args.task, components_manager=manager @@ -182,8 +201,9 @@ def main() -> int: if args.quantization == "int8": pipe = _apply_int8(pipe, args.quantizer_path) - if device == "mps" or (args.cpu_offload and args.quantization == "int8"): - _enable_offload(pipe, device=device, model_root=args.model_id) + if device == "mps" or disk_directory is not None or (args.cpu_offload and args.quantization == "int8"): + _enable_offload(pipe, device=device, model_root=args.model_id, + disk_directory=disk_directory.name if disk_directory else None) elif not args.cpu_offload: pipe.to(device) if device == "mps": @@ -259,6 +279,14 @@ def hook(module, inputs, output): print("MiniMax H3 推理失败: %s" % exc, file=sys.stderr) return 1 + finally: + if disk_directory is not None: + pipe = None + manager = None + import gc + gc.collect() + disk_directory.cleanup() + if __name__ == "__main__": sys.exit(main()) diff --git a/frontend/functional_ui/src/App.tsx b/frontend/functional_ui/src/App.tsx index 08421e31..5ed99bd8 100644 --- a/frontend/functional_ui/src/App.tsx +++ b/frontend/functional_ui/src/App.tsx @@ -1,5 +1,6 @@ import React from 'react'; import './App.css'; +import { ResourceMonitor } from './ui/common/ResourceMonitor'; import { UIStateProvider, connectAgent, agentDOMCommands } from 'ssui_components'; import { getUIProvider } from './ui/UIProvider'; import { SessionProvider } from './ui/conversation/SessionState'; @@ -36,7 +37,7 @@ function App() { }, [current.path, current.view]); if (!current.view) return
正在选择视图…
; const provider = getUIProvider(current.view) ?? getUIProvider('functional')!; - return
+ return
{provider.getUI(current.path)} diff --git a/frontend/functional_ui/src/ui/common/ResourceMonitor.tsx b/frontend/functional_ui/src/ui/common/ResourceMonitor.tsx new file mode 100644 index 00000000..2ed7a45b --- /dev/null +++ b/frontend/functional_ui/src/ui/common/ResourceMonitor.tsx @@ -0,0 +1,52 @@ +import { useEffect, useState } from 'react'; +import { Message } from 'ssui_components'; + +type Memory = { total: number; available: number; used: number }; +type Resources = { ram: Memory; gpus: (Memory & { index: number; name: string })[]; gpu_error: string | null }; +type DiskRequest = { id: string; model: string; required_bytes: number }; +const gib = (bytes: number) => (bytes / 2 ** 30).toFixed(1); + +export function ResourceMonitor() { + const [resources, setResources] = useState(); + const [request, setRequest] = useState(); + const [error, setError] = useState(''); + const [busy, setBusy] = useState(false); + useEffect(() => { + let disposed = false; + let timer: ReturnType; + const message = new Message(); + const poll = async () => { + try { + const [state, pending] = await Promise.all([ + message.get('api/resources/'), message.get('api/resources/disk-requests'), + ]); + if (!disposed) { setResources(state); setRequest(pending[0]); setError(''); } + } catch { if (!disposed) setError('资源监控连接失败'); } + if (!disposed) timer = setTimeout(poll, 2000); + }; + void poll(); + return () => { disposed = true; clearTimeout(timer); }; + }, []); + const answer = async (approved: boolean) => { + if (!request) return; + setBusy(true); + try { + await new Message().post(`api/resources/disk-requests/${request.id}`, { approved }); + setRequest(undefined); + } catch { setError('提交失败,请重试(请求可能已过期)'); } + finally { setBusy(false); } + }; + return ; +} diff --git a/frontend/ssui_components/src/api/generated.ts b/frontend/ssui_components/src/api/generated.ts index 94131d80..4937ad41 100644 --- a/frontend/ssui_components/src/api/generated.ts +++ b/frontend/ssui_components/src/api/generated.ts @@ -444,6 +444,59 @@ export interface paths { patch?: never; trace?: never; }; + "/api/resources/": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Resources */ + get: operations["resources_api_resources__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/resources/disk-requests": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Pending */ + get: operations["pending_api_resources_disk_requests_get"]; + put?: never; + /** Request Disk */ + post: operations["request_disk_api_resources_disk_requests_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/resources/disk-requests/{key}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Decision */ + get: operations["decision_api_resources_disk_requests__key__get"]; + put?: never; + /** Decide */ + post: operations["decide_api_resources_disk_requests__key__post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/script": { parameters: { query?: never; @@ -993,6 +1046,18 @@ export interface components { /** Prompt Param */ prompt_param: string; }; + /** Decision */ + Decision: { + /** Approved */ + approved: boolean; + }; + /** DiskRequest */ + DiskRequest: { + /** Model */ + model: string; + /** Required Bytes */ + required_bytes: number; + }; /** * DynamicResult * @description Executor-driven payload whose concrete shape is defined by a script. @@ -1883,6 +1948,145 @@ export interface operations { }; }; }; + resources_api_resources__get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; + pending_api_resources_disk_requests_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; + request_disk_api_resources_disk_requests_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["DiskRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + decision_api_resources_disk_requests__key__get: { + parameters: { + query?: never; + header?: never; + path: { + key: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + decide_api_resources_disk_requests__key__post: { + parameters: { + query?: never; + header?: never; + path: { + key: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["Decision"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; script_api_script_get: { parameters: { query: { diff --git a/server/app.py b/server/app.py index 893a07fd..2c34e438 100644 --- a/server/app.py +++ b/server/app.py @@ -16,6 +16,7 @@ from server.opener_service import FileOpenerManager from server.routes import ( config_routes, + resource_routes, extension_routes, file_routes, model_routes, @@ -88,6 +89,7 @@ async def lifespan(app: FastAPI): allow_headers=["*"], ) + app.include_router(resource_routes.router) app.include_router(config_routes.router) app.include_router(model_routes.router) app.include_router(script_routes.router) diff --git a/server/contract_app.py b/server/contract_app.py index e3faf7fc..987f63a9 100644 --- a/server/contract_app.py +++ b/server/contract_app.py @@ -12,6 +12,7 @@ from server.routes import ( config_routes, + resource_routes, extension_routes, file_routes, model_routes, @@ -23,6 +24,7 @@ def create_contract_app() -> FastAPI: app = FastAPI(title="SSUI API", version="1.0.0") + app.include_router(resource_routes.router) app.include_router(config_routes.router) app.include_router(model_routes.router) app.include_router(script_routes.router) diff --git a/server/openapi.json b/server/openapi.json index a8816281..bf753b34 100644 --- a/server/openapi.json +++ b/server/openapi.json @@ -349,6 +349,39 @@ "title": "ConversationInfo", "type": "object" }, + "Decision": { + "properties": { + "approved": { + "title": "Approved", + "type": "boolean" + } + }, + "required": [ + "approved" + ], + "title": "Decision", + "type": "object" + }, + "DiskRequest": { + "properties": { + "model": { + "maxLength": 512, + "title": "Model", + "type": "string" + }, + "required_bytes": { + "minimum": 0.0, + "title": "Required Bytes", + "type": "integer" + } + }, + "required": [ + "model", + "required_bytes" + ], + "title": "DiskRequest", + "type": "object" + }, "DynamicResult": { "description": "Executor-driven payload whose concrete shape is defined by a script.", "title": "DynamicResult" @@ -1758,6 +1791,154 @@ "summary": "Preset Models" } }, + "/api/resources/": { + "get": { + "operationId": "resources_api_resources__get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Resources" + } + }, + "/api/resources/disk-requests": { + "get": { + "operationId": "pending_api_resources_disk_requests_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Pending" + }, + "post": { + "operationId": "request_disk_api_resources_disk_requests_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DiskRequest" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Request Disk" + } + }, + "/api/resources/disk-requests/{key}": { + "get": { + "operationId": "decision_api_resources_disk_requests__key__get", + "parameters": [ + { + "in": "path", + "name": "key", + "required": true, + "schema": { + "title": "Key", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Decision" + }, + "post": { + "operationId": "decide_api_resources_disk_requests__key__post", + "parameters": [ + { + "in": "path", + "name": "key", + "required": true, + "schema": { + "title": "Key", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Decision" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Decide" + } + }, "/api/script": { "get": { "operationId": "script_api_script_get", diff --git a/server/routes/resource_routes.py b/server/routes/resource_routes.py new file mode 100644 index 00000000..cfc1bf39 --- /dev/null +++ b/server/routes/resource_routes.py @@ -0,0 +1,67 @@ +"""Short-lived, one-shot disk offload decisions shared with executor processes.""" +import time +import uuid +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field +from backend.resource_monitor import snapshot + +router = APIRouter(prefix="/api/resources") +_requests = {} + + +class DiskRequest(BaseModel): + model: str = Field(max_length=512) + required_bytes: int = Field(ge=0) + + +class Decision(BaseModel): + approved: bool + + +def _prune(): + now = time.monotonic() + for key in list(_requests): + if _requests[key]["expires"] <= now: + del _requests[key] + + +@router.get("/") +def resources(): + return snapshot() + + +@router.post("/disk-requests", status_code=201) +async def request_disk(body: DiskRequest): + _prune() + if len(_requests) >= 128: + raise HTTPException(429, "Too many pending offload requests") + key = str(uuid.uuid4()) + _requests[key] = {**body.model_dump(), "id": key, "approved": None, + "expires": time.monotonic() + 180} + return {"id": key} + + +@router.get("/disk-requests") +async def pending(): + _prune() + return [{k: v for k, v in r.items() if k != "expires"} + for r in _requests.values() if r["approved"] is None] + + +@router.get("/disk-requests/{key}") +async def decision(key: str): + _prune() + if key not in _requests: + raise HTTPException(404, "Offload request expired") + return {"approved": _requests[key]["approved"]} + + +@router.post("/disk-requests/{key}") +async def decide(key: str, body: Decision): + _prune() + if key not in _requests: + raise HTTPException(404, "Offload request expired") + if _requests[key]["approved"] is not None: + raise HTTPException(409, "Offload request already answered") + _requests[key]["approved"] = body.approved + return {"approved": body.approved} diff --git a/ssui/model_memory.py b/ssui/model_memory.py index d3c093f7..c86208c5 100644 --- a/ssui/model_memory.py +++ b/ssui/model_memory.py @@ -40,13 +40,16 @@ def configure(self, modules, device="cuda"): continue seen.add(id(module)) self.quantize(module) + from backend.runtime_offload import configure_pending + if configure_pending(module, device, self.cpu_offload): + continue if self.cpu_offload: offload_component(module, device) else: module.to(device) -def offload_component(module, device="cuda"): +def offload_component(module, device="cuda", *, cpu_state=None): """Keep a component on CPU between calls; release even on errors/close. Wrap custom inference methods as well as forward: several vendored models @@ -73,7 +76,10 @@ def resident(): depth -= 1 finally: if outer: - module.to("cpu") + if cpu_state is not None: + module.load_state_dict(cpu_state, assign=True) + else: + module.to("cpu") def move_inputs(args, kwargs): import torch @@ -116,7 +122,15 @@ def configure_diffsynth_manager(manager, options, device="cuda", *, quantize=Tru if quantize and options.quantization == "int8": for model in manager.model: options.quantize(model) - manager.to("cpu" if options.cpu_offload else device) + from backend.runtime_offload import configure_pending, is_managed, LoadPlan + if any(isinstance(getattr(m, '_ssui_load_plan', None), LoadPlan) for m in manager.model): + for model in manager.model: + if not configure_pending(model, device, options.cpu_offload): + model.to("cpu" if options.cpu_offload else device) + if any(is_managed(m) for m in manager.model): + options = ModelMemoryOptions(options.quantization, True, allow_fp8=True) + else: + manager.to("cpu" if options.cpu_offload else device) manager.device = str(device) manager.memory_options = options return manager diff --git a/tests/resource_monitor_test.py b/tests/resource_monitor_test.py new file mode 100644 index 00000000..73a3befa --- /dev/null +++ b/tests/resource_monitor_test.py @@ -0,0 +1,121 @@ +import unittest +import tempfile +import os +from unittest.mock import patch +from backend.resource_monitor import placement, GiB + + +class ResourcePolicyTests(unittest.TestCase): + def resources(self, ram=32, vram=24): + return {"ram": {"total": 64 * GiB, "available": ram * GiB}, + "gpus": [{"index": 0, "available": vram * GiB}]} + + def test_full_device_with_headroom(self): + self.assertEqual(placement(20 * GiB, self.resources()), "device") + + def test_low_vram_uses_host(self): + self.assertEqual(placement(22 * GiB, self.resources()), "cpu_offload") + + def test_host_pressure_requires_confirmation(self): + self.assertEqual(placement(22 * GiB, self.resources(ram=8)), "disk_confirmation") + + def test_missing_gpu_is_not_unlimited(self): + r = self.resources() + r["gpus"] = [] + self.assertEqual(placement(4 * GiB, r), "cpu_offload") + + def test_decline_and_unavailable_server_fail_closed(self): + from backend.disk_offload import confirm_disk_offload + with patch('backend.disk_offload.urlopen', side_effect=OSError('offline')): + with self.assertRaises(MemoryError): + confirm_disk_offload('test', GiB) + + def test_real_disk_backing_preserves_output(self): + import torch + from backend.disk_offload import disk_back_model + model = torch.nn.Linear(4, 3) + x = torch.randn(2, 4) + expected = model(x).detach().clone() + cache = tempfile.TemporaryDirectory() + with patch.dict(os.environ, {"SSUI_OFFLOAD_DIR": cache.name}): + directory = disk_back_model(model, 60) + try: + self.assertTrue(torch.equal(expected, model(x))) + model.load_state_dict({k: v.clone() for k, v in model.state_dict().items()}, assign=True) + finally: + directory.cleanup() + cache.cleanup() + + def test_api_confirmation_is_one_shot(self): + from fastapi import FastAPI + from fastapi.testclient import TestClient + from server.routes.resource_routes import router + app = FastAPI() + app.include_router(router) + with TestClient(app) as client: + key = client.post('/api/resources/disk-requests', json={ + 'model': 'test', 'required_bytes': GiB}).json()['id'] + self.assertIsNone(client.get('/api/resources/disk-requests/' + key).json()['approved']) + self.assertEqual(client.post('/api/resources/disk-requests/' + key, + json={'approved': False}).status_code, 200) + self.assertFalse(client.get('/api/resources/disk-requests/' + key).json()['approved']) + self.assertEqual(client.post('/api/resources/disk-requests/' + key, + json={'approved': True}).status_code, 409) + + def test_cache_uses_disk_mapping_only_after_consent(self): + import torch + try: + from backend.model_manager.load.model_cache.model_cache import ModelCache + except RuntimeError as exc: + if 'infer_schema(func)' in str(exc): + self.skipTest('Installed flash-attention is incompatible with torch: ' + str(exc).splitlines()[0]) + raise + from backend.model_manager.load.model_cache.cached_model.cached_model_with_partial_load import CachedModelWithPartialLoad + cache_root = tempfile.TemporaryDirectory() + resources = self.resources(ram=0, vram=0) + with patch.object(ModelCache, '_calc_ram_available_to_model_cache', return_value=GiB), \ + patch.object(ModelCache, 'make_room'), \ + patch('backend.resource_monitor.snapshot', return_value=resources), \ + patch('backend.disk_offload.confirm_disk_offload') as confirm, \ + patch.dict(os.environ, {'SSUI_OFFLOAD_DIR': cache_root.name}): + cache = ModelCache(3, True, True, execution_device='cuda') + model = torch.nn.Linear(4, 3) + expected = model.weight.detach().clone() + cache.put('tiny-model', model) + confirm.assert_called_once() + wrapped = cache.get('tiny-model').cached_model + self.assertIsInstance(wrapped, CachedModelWithPartialLoad) + self.assertTrue(torch.equal(expected, wrapped.get_cpu_state_dict()['weight'])) + backing = wrapped._disk_backing + model.load_state_dict({k: v.clone() for k, v in model.state_dict().items()}, assign=True) + wrapped._cpu_state_dict = None + backing.cleanup() + cache_root.cleanup() + + def test_confirmation_approval_and_refusal(self): + import io + import json + from backend.disk_offload import confirm_disk_offload + for approved in (True, False): + responses = [io.BytesIO(json.dumps({'id': 'one-shot'}).encode()), + io.BytesIO(json.dumps({'approved': approved}).encode())] + with patch('backend.disk_offload.urlopen', side_effect=responses): + if approved: + confirm_disk_offload('model', 1024) + else: + with self.assertRaises(MemoryError): + confirm_disk_offload('model', 1024) + + def test_low_host_requires_disk_even_if_gpu_fits(self): + self.assertEqual(placement(4 * GiB, self.resources(ram=1)), 'disk_confirmation') + + def test_disk_space_failure_does_not_create_backing_file(self): + import torch + from backend.disk_offload import disk_back_model + with tempfile.TemporaryDirectory() as root, \ + patch.dict(os.environ, {'SSUI_OFFLOAD_DIR': root}), \ + patch('backend.disk_offload.shutil.disk_usage') as usage: + usage.return_value.free = 0 + with self.assertRaises(MemoryError): + disk_back_model(torch.nn.Linear(1, 1), GiB) + self.assertEqual(os.listdir(root), []) diff --git a/tests/runtime_offload_test.py b/tests/runtime_offload_test.py new file mode 100644 index 00000000..52631ef5 --- /dev/null +++ b/tests/runtime_offload_test.py @@ -0,0 +1,238 @@ +"""Real CPU tensor lifecycle tests, independent of optional diffusion/CUDA packages.""" +import os +import tempfile +import unittest +from unittest.mock import patch +import torch +from backend.runtime_offload import LoadPlan, configure_model, plan_load, weight_size +from backend.resource_monitor import GiB + + +class RuntimeOffloadTests(unittest.TestCase): + def test_cpu_hooks_preserve_output_and_release_parameters(self): + model = torch.nn.Sequential(torch.nn.Linear(4, 8), torch.nn.ReLU(), torch.nn.Linear(8, 2)).eval() + x = torch.randn(3, 4) + expected = model(x).detach() + configure_model(model, LoadPlan('cpu_offload', 'cpu', 1000)) + self.assertTrue(all(p.device.type == 'meta' for p in model.parameters())) + with torch.no_grad(): + for _ in range(3): + self.assertTrue(torch.equal(expected, model(x))) + self.assertTrue(all(p.device.type == 'meta' for p in model.parameters())) + + def test_disk_hooks_preserve_output_and_use_mapped_cpu_storage(self): + model = torch.nn.Linear(3, 2).eval() + x = torch.randn(2, 3) + expected = model(x).detach() + with tempfile.TemporaryDirectory() as root, patch.dict(os.environ, {'SSUI_OFFLOAD_DIR': root}): + configure_model(model, LoadPlan('disk_offload', 'cpu', 100)) + self.assertTrue(os.path.isfile(os.path.join(model._ssui_disk_backing.name, 'weights.pt'))) + with torch.no_grad(): + self.assertTrue(torch.equal(expected, model(x))) + self.assertTrue(torch.equal(expected, model(x))) + from accelerate.hooks import remove_hook_from_module + remove_hook_from_module(model, recurse=True) + model.load_state_dict({k: v.clone() for k, v in model.state_dict().items()}, assign=True) + model._ssui_disk_backing.cleanup() + + def test_preflight_denial_happens_before_loader(self): + resources = {'ram': {'total': 16*GiB, 'available': GiB}, 'gpus': []} + with patch('backend.runtime_offload.snapshot', return_value=resources), \ + patch('backend.runtime_offload.confirm_disk_offload', side_effect=MemoryError('denied')) as confirm: + with self.assertRaises(MemoryError): + plan_load('video', estimated_bytes=8*GiB) + confirm.assert_called_once_with('video', 8*GiB) + + def test_explicit_disk_still_requires_consent(self): + resources = {'ram': {'total': 64*GiB, 'available': 60*GiB}, 'gpus': [{'index': 0, 'available': 24*GiB}]} + with patch('backend.runtime_offload.snapshot', return_value=resources), \ + patch('backend.runtime_offload.confirm_disk_offload') as confirm: + plan = plan_load('LTX', estimated_bytes=GiB, requested='disk') + self.assertEqual(plan.mode, 'disk_offload') + confirm.assert_called_once() + + def test_weight_files_counted_once(self): + with tempfile.TemporaryDirectory() as root: + from pathlib import Path + weight = Path(root) / 'model.safetensors' + weight.write_bytes(b'1234') + (Path(root) / 'config.json').write_text('{}') + self.assertEqual(weight_size([root, weight]), 4) + + def test_unknown_size_stops_before_model_load(self): + with self.assertRaises(MemoryError): + plan_load('unknown', '/not/a/local/weight') + + def test_diffsynth_actual_loader_keeps_execution_device_and_hooks_real_model(self): + import ast + from pathlib import Path + from types import SimpleNamespace + source = Path('extensions/Video/diffsynth/models/model_manager.py').read_text() + tree = ast.parse(source) + cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == 'ModelManager') + methods = [n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name in ('load_model', 'load_models')] + namespace = {'os': os, 'torch': torch, 'load_state_dict': lambda p: torch.load(p, weights_only=True)} + for method in methods: + exec(compile(ast.Module(body=[method], type_ignores=[]), '', 'exec'), namespace) + loaded = torch.nn.Linear(3, 2).eval() + detector = SimpleNamespace(match=lambda *args: True, + load=lambda *args, **kwargs: (['tiny'], [loaded])) + manager = SimpleNamespace(device='cpu', torch_dtype=torch.float32, + model=[], model_path=[], model_name=[], model_detector=[detector]) + import types + manager.load_model = types.MethodType(namespace['load_model'], manager) + manager.load_models = types.MethodType(namespace['load_models'], manager) + with tempfile.TemporaryDirectory() as root: + path = Path(root) / 'model.pt' + torch.save(loaded.state_dict(), path) + manager.load_models([str(path)]) + self.assertEqual(manager.device, 'cpu') + from backend.runtime_offload import configure_pending + configure_pending(manager.model[0], 'cpu') + self.assertEqual(manager.model[0]._ssui_residency, 'cpu_offload') + with torch.no_grad(): + self.assertEqual(manager.model[0](torch.ones(1, 3)).shape, (1, 2)) + + def test_trellis_pipeline_device_is_execution_device_not_meta(self): + import ast + from pathlib import Path + tree = ast.parse(Path('extensions/3D-Model/vendor/trellis/pipelines/base.py').read_text()) + cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == 'Pipeline') + namespace = {'torch': torch, 'nn': torch.nn} + exec(compile(ast.Module(body=[cls], type_ignores=[]), '', 'exec'), namespace) + model = torch.nn.Linear(3, 2).eval() + configure_model(model, LoadPlan('cpu_offload', 'cpu', 100)) + pipeline = namespace['Pipeline']({'tiny': model}) + pipeline.to('cpu') + self.assertEqual(pipeline.device, torch.device('cpu')) + self.assertTrue(all(p.device.type == 'meta' for p in model.parameters())) + with self.assertRaises(RuntimeError): + pipeline.to('cuda') + + def test_native_ltx_cli_receives_approved_effective_mode(self): + import ast + from pathlib import Path + tree = ast.parse(Path('extensions/Video/ssui_video/LTX2.py').read_text()) + fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == '_run_ltx') + import subprocess + namespace = {'_ltx_python': lambda: 'isolated-python', 'subprocess': subprocess, 'os': os, 'Path': Path, 'sys': __import__('sys'), '__file__': str(Path('extensions/Video/ssui_video/LTX2.py').resolve())} + exec(compile(ast.Module(body=[fn], type_ignores=[]), '', 'exec'), namespace) + with patch('backend.runtime_offload.plan_load', return_value=LoadPlan('disk_offload', 'cuda', GiB)) as planner, \ + patch('subprocess.Popen') as popen: + popen.return_value.stdout = [] + popen.return_value.returncode = 0 + namespace['_run_ltx']('generate', ['--transformer-path', '/weights/model', '--offload', 'cpu']) + command = popen.call_args.args[0] + self.assertEqual(command[-2:], ['--offload', 'disk']) + planner.assert_called_once_with('LTX-2.5', ['/weights/model'], requested='cpu') + + def test_failed_forward_releases_weights_for_retry(self): + class Failing(torch.nn.Module): + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(2)) + self.fail = True + def forward(self, x): + if self.fail: + raise ValueError('intentional failure') + return x * self.weight + model = Failing() + configure_model(model, LoadPlan('cpu_offload', 'cpu', 8)) + with self.assertRaisesRegex(ValueError, 'intentional'): + model(torch.ones(2)) + self.assertEqual(model.weight.device.type, 'meta') + model.fail = False + with torch.no_grad(): + self.assertTrue(torch.equal(model(torch.ones(2)), torch.ones(2))) + + def test_lora_style_updates_restore_and_reinstall_residency(self): + from backend.runtime_offload import materialize_for_update + class Manager: + def __init__(self, model): + self.model = [model] + @materialize_for_update + def update(self): + with torch.no_grad(): + self.model[0].weight.add_(1) + model = torch.nn.Linear(1, 1, bias=False) + with torch.no_grad(): + model.weight.zero_() + configure_model(model, LoadPlan('cpu_offload', 'cpu', 4)) + Manager(model).update() + self.assertEqual(model.weight.device.type, 'meta') + with torch.no_grad(): + self.assertTrue(torch.equal(model(torch.ones(1, 1)), torch.ones(1, 1))) + + def test_h3_disk_groups_have_separate_paths_and_no_stream_pinning(self): + import ast + from pathlib import Path + from types import SimpleNamespace, ModuleType + from unittest.mock import Mock + import sys + tree = ast.parse(Path('extensions/Video/ssui_video/h3_runner.py').read_text()) + fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == '_enable_offload') + namespace = {'Path': Path} + exec(compile(ast.Module(body=[fn], type_ignores=[]), '', 'exec'), namespace) + hooks = ModuleType('diffusers.hooks') + hooks.apply_group_offloading = Mock() + pipe = SimpleNamespace(transformer=Mock(), text_encoder=Mock(), vae=Mock(), audio_vae=Mock()) + with patch.dict(sys.modules, {'diffusers.hooks': hooks}): + namespace['_enable_offload'](pipe, disk_directory='/approved/offload') + transformer_options = pipe.transformer.enable_group_offload.call_args.kwargs + text_options = hooks.apply_group_offloading.call_args.kwargs + self.assertEqual(transformer_options['offload_to_disk_path'], '/approved/offload/transformer') + self.assertEqual(text_options['offload_to_disk_path'], '/approved/offload/text_encoder') + self.assertFalse(transformer_options['use_stream']) + + def test_trellis_actual_checkpoint_loader_uses_empty_init_and_real_hooks(self): + import ast + import json + from pathlib import Path + from safetensors.torch import save_file + tree = ast.parse(Path('extensions/3D-Model/vendor/trellis/models/__init__.py').read_text()) + fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'from_pretrained') + namespace = {'__getattr__': lambda name: torch.nn.Linear} + exec(compile(ast.Module(body=[fn], type_ignores=[]), '', 'exec'), namespace) + original = torch.nn.Linear(3, 2).eval() + x = torch.ones(1, 3) + expected = original(x).detach() + with tempfile.TemporaryDirectory() as root: + path = Path(root) / 'component' + path.with_suffix('.json').write_text(json.dumps({'name': 'Linear', 'args': {'in_features': 3, 'out_features': 2}})) + save_file(original.state_dict(), str(path.with_suffix('.safetensors'))) + with patch('backend.runtime_offload.plan_load', return_value=LoadPlan('cpu_offload', 'cpu', 32)) as planner: + loaded = namespace['from_pretrained'](str(path)) + planner.assert_called_once() + from backend.runtime_offload import configure_pending + configure_pending(loaded, 'cpu') + self.assertEqual(loaded.weight.device.type, 'meta') + with torch.no_grad(): + self.assertTrue(torch.equal(expected, loaded(x))) + + def test_deferred_native_int8_disk_policy_keeps_quantized_output(self): + from backend.runtime_offload import configure_pending + from ssui.native_quantization import quantize_model_int8 + model = torch.nn.Sequential(torch.nn.Linear(3, 2)).eval() + quantize_model_int8(model) + x = torch.ones(1, 3) + expected = model(x).detach() + model._ssui_load_plan = LoadPlan('disk_offload', 'cpu', 100) + with tempfile.TemporaryDirectory() as root, patch.dict(os.environ, {'SSUI_OFFLOAD_DIR': root}): + self.assertTrue(configure_pending(model, 'cpu')) + self.assertTrue(model._ssui_cpu_offload) + with torch.no_grad(): + self.assertTrue(torch.equal(expected, model(x))) + self.assertTrue(torch.equal(expected, model(x))) + model.load_state_dict({k: v.clone() for k, v in model.state_dict().items()}, assign=True) + model._ssui_disk_backing.cleanup() + + def test_fp8_preserves_native_casting_and_rejects_unsupported_disk(self): + from backend.runtime_offload import configure_pending + model = torch.nn.Linear(2, 2).to(dtype=torch.float8_e4m3fn) + model._ssui_load_plan = LoadPlan('cpu_offload', 'cpu', 6) + self.assertFalse(configure_pending(model, 'cpu')) + self.assertEqual(model.weight.dtype, torch.float8_e4m3fn) + self.assertFalse(hasattr(model, '_ssui_residency')) + model._ssui_load_plan = LoadPlan('disk_offload', 'cpu', 6) + with self.assertRaisesRegex(MemoryError, 'FP8'): + configure_pending(model, 'cpu') From 6b48a4fa68b2beef8941eb3784934490229d3f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A5=BF=E9=A3=8E=E9=80=8D=E9=81=A5=E6=B8=B8?= Date: Wed, 11 Nov 2026 23:26:34 +0800 Subject: [PATCH 2/2] fix: keep resource monitoring failures isolated and tests portable --- backend/resource_monitor.py | 3 +- .../src/ui/common/ResourceMonitor.tsx | 7 +++- tests/runtime_offload_test.py | 4 +-- tests/ui/fixtures.ts | 9 +++++ tests/ui/resource-monitor.spec.ts | 34 +++++++++++++++++++ 5 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 tests/ui/resource-monitor.spec.ts diff --git a/backend/resource_monitor.py b/backend/resource_monitor.py index 5d5c941b..a1656861 100644 --- a/backend/resource_monitor.py +++ b/backend/resource_monitor.py @@ -1,10 +1,11 @@ """Machine resources and conservative model placement decisions (all sizes are bytes).""" -import psutil GiB = 2**30 def snapshot(): + # Schema-only imports must not require optional monitoring dependencies. + import psutil ram = psutil.virtual_memory() result = {"ram": {"total": ram.total, "available": ram.available, "used": ram.total - ram.available}, "gpus": [], "gpu_error": None} diff --git a/frontend/functional_ui/src/ui/common/ResourceMonitor.tsx b/frontend/functional_ui/src/ui/common/ResourceMonitor.tsx index 2ed7a45b..1abaa4d4 100644 --- a/frontend/functional_ui/src/ui/common/ResourceMonitor.tsx +++ b/frontend/functional_ui/src/ui/common/ResourceMonitor.tsx @@ -4,6 +4,11 @@ import { Message } from 'ssui_components'; type Memory = { total: number; available: number; used: number }; type Resources = { ram: Memory; gpus: (Memory & { index: number; name: string })[]; gpu_error: string | null }; type DiskRequest = { id: string; model: string; required_bytes: number }; +async function getResource(message: Message, path: string) { + const response = await fetch(`http://${message.host}:${message.port}/api/resources/${path}`); + if (!response.ok) throw new Error(`Resource monitor HTTP ${response.status}`); + return response.json(); +} const gib = (bytes: number) => (bytes / 2 ** 30).toFixed(1); export function ResourceMonitor() { @@ -18,7 +23,7 @@ export function ResourceMonitor() { const poll = async () => { try { const [state, pending] = await Promise.all([ - message.get('api/resources/'), message.get('api/resources/disk-requests'), + getResource(message, ''), getResource(message, 'disk-requests'), ]); if (!disposed) { setResources(state); setRequest(pending[0]); setError(''); } } catch { if (!disposed) setError('资源监控连接失败'); } diff --git a/tests/runtime_offload_test.py b/tests/runtime_offload_test.py index 52631ef5..dfb9e118 100644 --- a/tests/runtime_offload_test.py +++ b/tests/runtime_offload_test.py @@ -180,8 +180,8 @@ def test_h3_disk_groups_have_separate_paths_and_no_stream_pinning(self): namespace['_enable_offload'](pipe, disk_directory='/approved/offload') transformer_options = pipe.transformer.enable_group_offload.call_args.kwargs text_options = hooks.apply_group_offloading.call_args.kwargs - self.assertEqual(transformer_options['offload_to_disk_path'], '/approved/offload/transformer') - self.assertEqual(text_options['offload_to_disk_path'], '/approved/offload/text_encoder') + self.assertEqual(Path(transformer_options['offload_to_disk_path']), Path('/approved/offload') / 'transformer') + self.assertEqual(Path(text_options['offload_to_disk_path']), Path('/approved/offload') / 'text_encoder') self.assertFalse(transformer_options['use_stream']) def test_trellis_actual_checkpoint_loader_uses_empty_init_and_real_hooks(self): diff --git a/tests/ui/fixtures.ts b/tests/ui/fixtures.ts index fc7b215f..a162f87c 100644 --- a/tests/ui/fixtures.ts +++ b/tests/ui/fixtures.ts @@ -5,6 +5,11 @@ export const defaults = { external_code_editor: '', civitai_token: '', }; +export const resources = { + ram: { total: 16 * 2 ** 30, available: 8 * 2 ** 30, used: 8 * 2 ** 30 }, + gpus: [], gpu_error: null, +}; + // Real application/React/Monaco; only the service boundary is simulated. // Unexpected API calls fail the test, so fixtures cannot silently hide regressions. export const test = base.extend<{ serviceGuard: void }>({ @@ -24,6 +29,10 @@ export const test = base.extend<{ serviceGuard: void }>({ } return route.fulfill({ json: states.get(key) ?? {} }); } + if (route.request().method() === 'GET' && path === '/api/resources/') + return route.fulfill({ json: resources }); + if (route.request().method() === 'GET' && path === '/api/resources/disk-requests') + return route.fulfill({ json: [] }); if (path === '/config/') return route.fulfill({ json: { ui: defaults } }); unexpected.push(`${route.request().method()} ${path}`); await route.fulfill({ status: 501, json: { error: 'Unmocked API' } }); diff --git a/tests/ui/resource-monitor.spec.ts b/tests/ui/resource-monitor.spec.ts new file mode 100644 index 00000000..63b52a56 --- /dev/null +++ b/tests/ui/resource-monitor.spec.ts @@ -0,0 +1,34 @@ +import { test, expect, resources, view } from './fixtures'; + +test('resource HTTP failure keeps the current view usable and polling recovers', async ({ page }) => { + let healthy = false; + await page.route('**/api/resources/', route => healthy + ? route.fulfill({ json: resources }) + : route.fulfill({ status: 503, json: { error: 'monitor temporarily unavailable' } })); + await page.goto(view('project_settings')); + await expect(page.getByRole('status').filter({ hasText: '资源监控连接失败' })).toBeVisible(); + await expect(page.getByLabel('主题颜色')).toBeVisible(); + healthy = true; + await expect(page.getByLabel('机器资源')).toContainText('内存 8.0 / 16.0 GiB'); + await expect(page.getByText('资源监控连接失败')).toHaveCount(0); +}); + +test('disk confirmation posts the explicit decision and clears after success', async ({ page }) => { + let pending = true; + let decision: unknown; + await page.route('**/api/resources/disk-requests', route => route.fulfill({ json: pending + ? [{ id: 'test-offload', model: 'example-model', required_bytes: 4 * 2 ** 30 }] + : [] })); + await page.route('**/api/resources/disk-requests/test-offload', route => { + decision = route.request().postDataJSON(); + pending = false; + return route.fulfill({ json: decision }); + }); + await page.goto(view('project_settings')); + const dialog = page.getByRole('dialog', { name: '磁盘 offload 确认' }); + await expect(dialog).toContainText('example-model'); + await dialog.getByRole('button', { name: '拒绝并停止' }).click(); + await expect(dialog).toHaveCount(0); + expect(decision).toEqual({ approved: false }); + await expect(page.getByLabel('主题颜色')).toBeVisible(); +});