From 5c9f675db9e852190b97dd3080315f8239634557 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Wed, 18 Nov 2026 10:33:44 +0100 Subject: [PATCH] fix(models): never treat a truncated weight download as complete http.client returns b"" when a server closes the stream early instead of raising, so _download_file_streamed renamed a short .part file into place and the model looked installed while its weights could never load (#389). Downloads now check the received size against the size Hugging Face lists for the file (one list_repo_tree call per repo) or the response length, and resume from the .part file when it falls short. An existing file whose size does not match is fetched again instead of being skipped. For models already broken on disk, a load that fails on a truncated safetensors header or torch archive now reports a short message pointing to the Models page instead of the raw traceback. --- api/routers/generation.py | 4 +- api/routers/model.py | 71 ++++++++++++---- api/services/generator_registry.py | 11 ++- api/services/weight_integrity.py | 38 +++++++++ api/tests/test_model_router.py | 123 ++++++++++++++++++++++++++- api/tests/test_weight_integrity.py | 128 +++++++++++++++++++++++++++++ 6 files changed, 354 insertions(+), 21 deletions(-) create mode 100644 api/services/weight_integrity.py create mode 100644 api/tests/test_weight_integrity.py diff --git a/api/routers/generation.py b/api/routers/generation.py index dc8ad3183..c15d07152 100644 --- a/api/routers/generation.py +++ b/api/routers/generation.py @@ -18,6 +18,7 @@ from services.generator_registry import generator_registry from schemas.generation import GenerateFromArtifactRequest, JobStatus from services.run_history import run_history +from services.weight_integrity import IncompleteModelWeightsError from services.artifact_input import ( RESERVED_ARTIFACT_PARAMS, TypedArtifactInput, @@ -453,7 +454,8 @@ def progress_cb(pct: int, step: str = "") -> None: except UnicodeEncodeError: print(msg.encode("ascii", errors="replace").decode("ascii")) job.status = "error" - job.error = tb.strip() + # Actionable errors reach the user as-is; the full traceback stays in the log. + job.error = str(exc) if isinstance(exc, IncompleteModelWeightsError) else tb.strip() _completed_at[job_id] = time.monotonic() finally: _job_generators.pop(job_id, None) diff --git a/api/routers/model.py b/api/routers/model.py index aab4c3f4c..ca0a88f4c 100644 --- a/api/routers/model.py +++ b/api/routers/model.py @@ -184,26 +184,25 @@ def _fmt(data: dict) -> str: try: yield _fmt({"percent": 0, "status": "Listing repository files..."}) files_by_source: dict[str, list[str]] = {} + sizes_by_source: dict[str, dict[str, Optional[int]]] = {} for source in sources: _check_download_control(control) def _list_files(current=source): - from huggingface_hub import list_repo_files - listed = list_repo_files( - current["repo_id"], - revision=current.get("revision"), - token=hf_token, + listed = _list_repo_file_sizes( + current["repo_id"], current.get("revision"), hf_token ) include = current.get("include_prefixes", []) skip = current.get("skip_prefixes", []) - return [ - filename for filename in listed + return { + filename: size for filename, size in listed.items() if (not include or any(filename.startswith(prefix) for prefix in include)) if not any(filename.startswith(prefix) for prefix in skip) - ] + } - files = await loop.run_in_executor(None, _list_files) + listed_sizes = await loop.run_in_executor(None, _list_files) + files = list(listed_sizes) if not files: raise RuntimeError( f'No files found in Hugging Face repo: {source["repo_id"]}' @@ -212,6 +211,7 @@ def _list_files(current=source): for filename in files: resolve_download_path(destination, filename) files_by_source[source["id"]] = files + sizes_by_source[source["id"]] = listed_sizes validate_source_file_plan(sources, files_by_source) planned_files = [ @@ -261,6 +261,7 @@ def _progress(message: dict) -> None: progress_cb=_progress, control=control, token=hf_token, + expected_size=sizes_by_source[source["id"]].get(filename), ), ) while not future.done(): @@ -364,14 +365,14 @@ def _fmt(data: dict) -> str: _check_download_control(control) def _list_files(): - from huggingface_hub import list_repo_files - return [ - f for f in list_repo_files(repo_id, token=hf_token) + return { + f: size for f, size in _list_repo_file_sizes(repo_id, None, hf_token).items() if (not include_list or any(f.startswith(p) for p in include_list)) if not any(f.startswith(p) for p in skip_list) - ] + } - files = await loop.run_in_executor(None, _list_files) + file_sizes = await loop.run_in_executor(None, _list_files) + files = list(file_sizes) total = len(files) if total == 0: @@ -413,6 +414,7 @@ def _progress(msg: dict) -> None: progress_cb=_progress, control=control, token=hf_token, + expected_size=file_sizes.get(filename), ), ) @@ -459,6 +461,30 @@ def _progress(msg: dict) -> None: return StreamingResponse(stream(), media_type="text/event-stream") +class IncompleteDownload(OSError): + """The server closed the stream before the whole file arrived. + + An OSError so the retry loop resumes from the .part file. http.client does + not raise on a short body (read() returns b"" at EOF), so without this check + a truncated download was renamed into place and later treated as complete. + """ + + +def _list_repo_file_sizes( + repo_id: str, revision: Optional[str], token: Optional[str] +) -> dict[str, Optional[int]]: + """Every file of a Hugging Face repo with its size in bytes, in one API call.""" + from huggingface_hub import HfApi + return { + entry.path: entry.size + for entry in HfApi().list_repo_tree( + repo_id, revision=revision, token=token, recursive=True + ) + # RepoFolder entries have no size; RepoFile entries always do. + if hasattr(entry, "size") + } + + def _download_file_streamed( *, url: str, @@ -470,6 +496,7 @@ def _download_file_streamed( progress_cb, control: dict[str, threading.Event], token: Optional[str] = None, + expected_size: Optional[int] = None, ) -> int: final_path = Path(dest_dir) / filename temp_path = final_path.with_suffix(final_path.suffix + ".part") @@ -479,8 +506,9 @@ def _download_file_streamed( if not final_path.is_file(): raise RuntimeError(f"Download target is not a regular file: {filename}") existing_size = final_path.stat().st_size - if existing_size > 0: + if existing_size > 0 and (expected_size is None or existing_size == expected_size): return existing_size + # Empty, or truncated by an older build: fetch it again. final_path.unlink() # Explicit token (from caller) > env vars > none @@ -502,6 +530,12 @@ def _download_file_streamed( try: _check_download_control(control) existing_bytes = temp_path.stat().st_size if temp_path.exists() else 0 + if expected_size is not None and existing_bytes > expected_size: + temp_path.unlink() + existing_bytes = 0 + if expected_size is not None and existing_bytes == expected_size and existing_bytes > 0: + temp_path.replace(final_path) + return existing_bytes request_headers = dict(headers) request_url = url if existing_bytes > 0: @@ -560,6 +594,13 @@ def _download_file_streamed( }) last_emit = now + expected_bytes = expected_size if expected_size is not None else total_bytes + if expected_bytes is not None and bytes_downloaded != expected_bytes: + if bytes_downloaded > expected_bytes: + temp_path.unlink(missing_ok=True) + raise IncompleteDownload( + f"{filename}: received {bytes_downloaded} of {expected_bytes} bytes" + ) temp_path.replace(final_path) return bytes_downloaded diff --git a/api/services/generator_registry.py b/api/services/generator_registry.py index 40735ce19..ff153436f 100644 --- a/api/services/generator_registry.py +++ b/api/services/generator_registry.py @@ -25,6 +25,7 @@ from services.generators.base import BaseGenerator from services.extension_process import ExtensionProcess, _venv_python +from services.weight_integrity import IncompleteModelWeightsError, incomplete_weights_message from services.model_sources import ( missing_weight_variant, model_sources_are_downloaded, @@ -837,7 +838,15 @@ def get_ready_generator(self, model_id: str) -> BaseGenerator: pass else: gen._auto_download() - gen.load() + try: + gen.load() + except Exception as exc: + message = incomplete_weights_message( + manifest.get("name", gen.DISPLAY_NAME), exc + ) + if message is None: + raise + raise IncompleteModelWeightsError(message) from exc return gen def activate_ready_generator(self, model_id: str) -> BaseGenerator: diff --git a/api/services/weight_integrity.py b/api/services/weight_integrity.py new file mode 100644 index 000000000..2ab6b88be --- /dev/null +++ b/api/services/weight_integrity.py @@ -0,0 +1,38 @@ +"""Recognise a model load that failed because a weight file on disk is truncated. + +Older builds could rename a partially downloaded file into place, and the model +then looked installed while its weights could never load. The loader's own +error is a deep traceback; this turns it into something the user can act on. +""" +import re +from typing import Optional + +# Substrings that only appear when a weight file's bytes are cut short or garbled. +_TRUNCATED_WEIGHT_SIGNATURES = ( + # safetensors: header longer than the file, or a file shorter than its header says + "Error while deserializing header", + # torch.load on a cut-off .bin/.pt/.ckpt zip archive + "failed finding central directory", +) + +_CHECKPOINT_PATH = re.compile( + r"Unable to load weights from checkpoint file for '([^']+)'" +) + + +class IncompleteModelWeightsError(RuntimeError): + """A model's weight files on disk are incomplete; its message is user-facing.""" + + +def incomplete_weights_message(model_name: str, error: BaseException) -> Optional[str]: + """A short, actionable message when ``error`` comes from truncated weights, else None.""" + text = str(error) + if not any(signature in text for signature in _TRUNCATED_WEIGHT_SIGNATURES): + return None + match = _CHECKPOINT_PATH.search(text) + detail = f" ({match.group(1)})" if match else "" + return ( + f"The model files for {model_name} are incomplete or corrupted{detail}, " + "most likely from an interrupted download. Open the Models page, remove " + "this node's weights, then install them again." + ) diff --git a/api/tests/test_model_router.py b/api/tests/test_model_router.py index 4090a0ea0..0a90f3f10 100644 --- a/api/tests/test_model_router.py +++ b/api/tests/test_model_router.py @@ -86,14 +86,15 @@ def tearDown(self) -> None: def install_hf_stub(self, files: dict[str, list[str]], calls: list[str]) -> None: module = types.ModuleType("huggingface_hub") - def list_repo_files(repo_id, revision=None, token=None): - calls.append(f"list:{repo_id}:{revision}:{token}") - return files[repo_id] + class HfApi: + def list_repo_tree(self, repo_id, revision=None, token=None, recursive=False): + calls.append(f"list:{repo_id}:{revision}:{token}") + return [types.SimpleNamespace(path=name, size=None) for name in files[repo_id]] def hf_hub_url(repo_id, filename, revision=None): return f"https://example.invalid/{repo_id}/{revision or 'main'}/{filename}" - module.list_repo_files = list_repo_files + module.HfApi = HfApi module.hf_hub_url = hf_hub_url sys.modules["huggingface_hub"] = module @@ -309,5 +310,119 @@ async def run(): self.assertEqual(switched, {"active": "demo/b"}) +class _FakeResponse: + """urlopen() stand-in serving ``body`` and, like http.client, returning b"" + at EOF even when Content-Length promised more.""" + + def __init__(self, body: bytes, content_length: int, status: int = 200) -> None: + self._body = body + self.status = status + self.headers = {"Content-Length": str(content_length)} + + def __enter__(self): + return self + + def __exit__(self, *exc) -> None: + return None + + def read(self, size: int) -> bytes: + chunk, self._body = self._body[:size], self._body[size:] + return chunk + + +class DownloadFileStreamedTests(unittest.TestCase): + FULL = b"0123456789" + + def setUp(self) -> None: + self.tempdir = tempfile.TemporaryDirectory(prefix="modly-download-") + self.dest = Path(self.tempdir.name) + self.control = {"pause": threading.Event(), "cancel": threading.Event()} + self.requests: list[dict] = [] + patches = [ + patch.object(model_router.time, "sleep", lambda _s: None), + patch.object(model_router, "_resolve_direct_download_url", lambda url, headers: url), + ] + for p in patches: + p.start() + self.addCleanup(p.stop) + + def tearDown(self) -> None: + self.tempdir.cleanup() + + def download(self, responses: list, expected_size=None) -> int: + def fake_urlopen(request, timeout=None): + self.requests.append(dict(request.header_items())) + return responses.pop(0) + + with patch.object(model_router, "urlopen", fake_urlopen): + return model_router._download_file_streamed( + url="https://example.invalid/w.safetensors", + filename="w.safetensors", + dest_dir=str(self.dest), + file_index=1, + total_files=1, + base_percent=1, + progress_cb=lambda _m: None, + control=self.control, + expected_size=expected_size, + ) + + def test_stream_closed_early_resumes_instead_of_saving_a_truncated_file(self) -> None: + size = self.download([ + _FakeResponse(self.FULL[:4], content_length=len(self.FULL)), + _FakeResponse(self.FULL[4:], content_length=6, status=206), + ]) + + self.assertEqual(size, len(self.FULL)) + self.assertEqual((self.dest / "w.safetensors").read_bytes(), self.FULL) + self.assertEqual(self.requests[1].get("Range"), "bytes=4-") + self.assertFalse((self.dest / "w.safetensors.part").exists()) + + def test_never_renames_a_short_file_into_place(self) -> None: + responses = [_FakeResponse(self.FULL[:4], content_length=len(self.FULL))] + responses += [_FakeResponse(b"", content_length=6, status=206) for _ in range(2)] + with self.assertRaisesRegex(RuntimeError, "received 4 of 10 bytes"): + self.download(responses) + + self.assertFalse((self.dest / "w.safetensors").exists()) + self.assertEqual((self.dest / "w.safetensors.part").read_bytes(), self.FULL[:4]) + + def test_existing_file_with_the_wrong_size_is_downloaded_again(self) -> None: + (self.dest / "w.safetensors").write_bytes(self.FULL[:4]) + size = self.download( + [_FakeResponse(self.FULL, content_length=len(self.FULL))], + expected_size=len(self.FULL), + ) + + self.assertEqual(size, len(self.FULL)) + self.assertEqual((self.dest / "w.safetensors").read_bytes(), self.FULL) + + def test_existing_file_with_the_expected_size_is_kept(self) -> None: + (self.dest / "w.safetensors").write_bytes(self.FULL) + self.assertEqual(self.download([], expected_size=len(self.FULL)), len(self.FULL)) + self.assertEqual(self.requests, []) + + def test_repo_listing_reports_file_sizes_and_skips_folders(self) -> None: + module = types.ModuleType("huggingface_hub") + seen: dict = {} + + class HfApi: + def list_repo_tree(self, repo_id, revision=None, token=None, recursive=False): + seen.update(repo_id=repo_id, revision=revision, token=token, recursive=recursive) + return [ + types.SimpleNamespace(path="transformer", tree_id="t"), + types.SimpleNamespace(path="transformer/w.safetensors", size=10), + ] + + module.HfApi = HfApi + with patch.dict(sys.modules, {"huggingface_hub": module}): + sizes = model_router._list_repo_file_sizes("org/repo", "main", "tok") + + self.assertEqual(sizes, {"transformer/w.safetensors": 10}) + self.assertEqual( + seen, {"repo_id": "org/repo", "revision": "main", "token": "tok", "recursive": True} + ) + + if __name__ == "__main__": unittest.main() diff --git a/api/tests/test_weight_integrity.py b/api/tests/test_weight_integrity.py new file mode 100644 index 000000000..792e56ab3 --- /dev/null +++ b/api/tests/test_weight_integrity.py @@ -0,0 +1,128 @@ +import asyncio +import tempfile +import threading +import unittest +from pathlib import Path + +import routers.generation as generation +import services.generator_registry as registry +from schemas.generation import JobStatus +from services.generator_registry import GeneratorRegistry +from services.weight_integrity import ( + IncompleteModelWeightsError, + incomplete_weights_message, +) + +# The tail of the traceback reported in issue #389 (TripoSG, diffusers). +TRUNCATED_SAFETENSORS = ( + "Traceback (most recent call last):\n" + "safetensors._safetensors_rust.SafetensorError: Error while deserializing header: " + "incomplete metadata, file not fully covered\n" + "OSError: Unable to load weights from checkpoint file for " + "'C:\\models\\triposg\\generate\\transformer\\diffusion_pytorch_model.safetensors' " + "at 'C:\\models\\triposg\\generate\\transformer\\diffusion_pytorch_model.safetensors'." +) + + +class _FailingLoadGenerator: + DISPLAY_NAME = "Fallback" + VRAM_GB = 1 + + def __init__(self, error: Exception) -> None: + self.error = error + + def is_downloaded(self) -> bool: + return True + + def is_loaded(self) -> bool: + return False + + def load(self) -> None: + raise self.error + + +class IncompleteWeightsMessageTests(unittest.TestCase): + def test_truncated_safetensors_names_the_file_and_the_fix(self) -> None: + message = incomplete_weights_message("TripoSG", RuntimeError(TRUNCATED_SAFETENSORS)) + self.assertIn("model files for TripoSG are incomplete", message) + self.assertIn("transformer\\diffusion_pytorch_model.safetensors", message) + self.assertIn("Models page", message) + self.assertNotIn("Traceback", message) + + def test_truncated_torch_archive_is_recognised(self) -> None: + error = RuntimeError( + "PytorchStreamReader failed reading zip archive: failed finding central directory" + ) + self.assertIsNotNone(incomplete_weights_message("Demo", error)) + + def test_unrelated_load_errors_are_left_alone(self) -> None: + self.assertIsNone( + incomplete_weights_message("Demo", RuntimeError("CUDA out of memory")) + ) + + +class RegistryLoadTests(unittest.TestCase): + def _registry(self, error: Exception) -> GeneratorRegistry: + reg = GeneratorRegistry() + reg._generators["triposg/generate"] = _FailingLoadGenerator(error) + reg._manifests["triposg/generate"] = {"name": "TripoSG"} + return reg + + def test_truncated_weights_raise_a_short_actionable_error(self) -> None: + reg = self._registry(RuntimeError(TRUNCATED_SAFETENSORS)) + with self.assertRaises(IncompleteModelWeightsError) as raised: + reg.get_ready_generator("triposg/generate") + self.assertIn("TripoSG", str(raised.exception)) + self.assertIsInstance(raised.exception.__cause__, RuntimeError) + + def test_other_load_errors_propagate_unchanged(self) -> None: + original = RuntimeError("CUDA out of memory") + reg = self._registry(original) + with self.assertRaises(RuntimeError) as raised: + reg.get_ready_generator("triposg/generate") + self.assertIs(raised.exception, original) + + +class GenerationJobErrorTests(unittest.TestCase): + def setUp(self) -> None: + self._prev_registry = generation.generator_registry + self._prev_ws = registry.WORKSPACE_DIR + self._tmp = tempfile.TemporaryDirectory() + registry.WORKSPACE_DIR = Path(self._tmp.name) + + def tearDown(self) -> None: + generation.generator_registry = self._prev_registry + registry.WORKSPACE_DIR = self._prev_ws + for store in ( + generation._jobs, + generation._cancel_events, + generation._cancelled, + generation._completed_at, + ): + store.clear() + self._tmp.cleanup() + + def test_job_error_is_the_short_message_not_the_traceback(self) -> None: + class _Registry: + def assert_weight_variant_installed(self, params, model_id=None) -> None: + pass + + def active_status(self) -> dict: + return {"loaded": True, "name": "TripoSG", "downloaded": True} + + def get_active(self): + raise IncompleteModelWeightsError("The model files for TripoSG are incomplete.") + + generation.generator_registry = _Registry() + job_id = "job-incomplete-weights" + generation._jobs[job_id] = JobStatus(job_id=job_id, status="pending", progress=0) + generation._cancel_events[job_id] = threading.Event() + asyncio.run(generation._run_generation(job_id, b"img", {}, "Coll")) + + job = generation._jobs[job_id] + self.assertEqual(job.status, "error") + self.assertEqual(job.error, "The model files for TripoSG are incomplete.") + + +if __name__ == "__main__": + unittest.main()