diff --git a/api/routers/generation.py b/api/routers/generation.py index dc8ad318..c15d0715 100644 --- a/api/routers/generation.py +++ b/api/routers/generation.py @@ -18,6 +18,7 @@ from services.generator_registry import generator_registry from schemas.generation import GenerateFromArtifactRequest, JobStatus from services.run_history import run_history +from services.weight_integrity import IncompleteModelWeightsError from services.artifact_input import ( RESERVED_ARTIFACT_PARAMS, TypedArtifactInput, @@ -453,7 +454,8 @@ def progress_cb(pct: int, step: str = "") -> None: except UnicodeEncodeError: print(msg.encode("ascii", errors="replace").decode("ascii")) job.status = "error" - job.error = tb.strip() + # Actionable errors reach the user as-is; the full traceback stays in the log. + job.error = str(exc) if isinstance(exc, IncompleteModelWeightsError) else tb.strip() _completed_at[job_id] = time.monotonic() finally: _job_generators.pop(job_id, None) diff --git a/api/routers/model.py b/api/routers/model.py index aab4c3f4..ca0a88f4 100644 --- a/api/routers/model.py +++ b/api/routers/model.py @@ -184,26 +184,25 @@ def _fmt(data: dict) -> str: try: yield _fmt({"percent": 0, "status": "Listing repository files..."}) files_by_source: dict[str, list[str]] = {} + sizes_by_source: dict[str, dict[str, Optional[int]]] = {} for source in sources: _check_download_control(control) def _list_files(current=source): - from huggingface_hub import list_repo_files - listed = list_repo_files( - current["repo_id"], - revision=current.get("revision"), - token=hf_token, + listed = _list_repo_file_sizes( + current["repo_id"], current.get("revision"), hf_token ) include = current.get("include_prefixes", []) skip = current.get("skip_prefixes", []) - return [ - filename for filename in listed + return { + filename: size for filename, size in listed.items() if (not include or any(filename.startswith(prefix) for prefix in include)) if not any(filename.startswith(prefix) for prefix in skip) - ] + } - files = await loop.run_in_executor(None, _list_files) + listed_sizes = await loop.run_in_executor(None, _list_files) + files = list(listed_sizes) if not files: raise RuntimeError( f'No files found in Hugging Face repo: {source["repo_id"]}' @@ -212,6 +211,7 @@ def _list_files(current=source): for filename in files: resolve_download_path(destination, filename) files_by_source[source["id"]] = files + sizes_by_source[source["id"]] = listed_sizes validate_source_file_plan(sources, files_by_source) planned_files = [ @@ -261,6 +261,7 @@ def _progress(message: dict) -> None: progress_cb=_progress, control=control, token=hf_token, + expected_size=sizes_by_source[source["id"]].get(filename), ), ) while not future.done(): @@ -364,14 +365,14 @@ def _fmt(data: dict) -> str: _check_download_control(control) def _list_files(): - from huggingface_hub import list_repo_files - return [ - f for f in list_repo_files(repo_id, token=hf_token) + return { + f: size for f, size in _list_repo_file_sizes(repo_id, None, hf_token).items() if (not include_list or any(f.startswith(p) for p in include_list)) if not any(f.startswith(p) for p in skip_list) - ] + } - files = await loop.run_in_executor(None, _list_files) + file_sizes = await loop.run_in_executor(None, _list_files) + files = list(file_sizes) total = len(files) if total == 0: @@ -413,6 +414,7 @@ def _progress(msg: dict) -> None: progress_cb=_progress, control=control, token=hf_token, + expected_size=file_sizes.get(filename), ), ) @@ -459,6 +461,30 @@ def _progress(msg: dict) -> None: return StreamingResponse(stream(), media_type="text/event-stream") +class IncompleteDownload(OSError): + """The server closed the stream before the whole file arrived. + + An OSError so the retry loop resumes from the .part file. http.client does + not raise on a short body (read() returns b"" at EOF), so without this check + a truncated download was renamed into place and later treated as complete. + """ + + +def _list_repo_file_sizes( + repo_id: str, revision: Optional[str], token: Optional[str] +) -> dict[str, Optional[int]]: + """Every file of a Hugging Face repo with its size in bytes, in one API call.""" + from huggingface_hub import HfApi + return { + entry.path: entry.size + for entry in HfApi().list_repo_tree( + repo_id, revision=revision, token=token, recursive=True + ) + # RepoFolder entries have no size; RepoFile entries always do. + if hasattr(entry, "size") + } + + def _download_file_streamed( *, url: str, @@ -470,6 +496,7 @@ def _download_file_streamed( progress_cb, control: dict[str, threading.Event], token: Optional[str] = None, + expected_size: Optional[int] = None, ) -> int: final_path = Path(dest_dir) / filename temp_path = final_path.with_suffix(final_path.suffix + ".part") @@ -479,8 +506,9 @@ def _download_file_streamed( if not final_path.is_file(): raise RuntimeError(f"Download target is not a regular file: {filename}") existing_size = final_path.stat().st_size - if existing_size > 0: + if existing_size > 0 and (expected_size is None or existing_size == expected_size): return existing_size + # Empty, or truncated by an older build: fetch it again. final_path.unlink() # Explicit token (from caller) > env vars > none @@ -502,6 +530,12 @@ def _download_file_streamed( try: _check_download_control(control) existing_bytes = temp_path.stat().st_size if temp_path.exists() else 0 + if expected_size is not None and existing_bytes > expected_size: + temp_path.unlink() + existing_bytes = 0 + if expected_size is not None and existing_bytes == expected_size and existing_bytes > 0: + temp_path.replace(final_path) + return existing_bytes request_headers = dict(headers) request_url = url if existing_bytes > 0: @@ -560,6 +594,13 @@ def _download_file_streamed( }) last_emit = now + expected_bytes = expected_size if expected_size is not None else total_bytes + if expected_bytes is not None and bytes_downloaded != expected_bytes: + if bytes_downloaded > expected_bytes: + temp_path.unlink(missing_ok=True) + raise IncompleteDownload( + f"{filename}: received {bytes_downloaded} of {expected_bytes} bytes" + ) temp_path.replace(final_path) return bytes_downloaded diff --git a/api/services/generator_registry.py b/api/services/generator_registry.py index 40735ce1..ff153436 100644 --- a/api/services/generator_registry.py +++ b/api/services/generator_registry.py @@ -25,6 +25,7 @@ from services.generators.base import BaseGenerator from services.extension_process import ExtensionProcess, _venv_python +from services.weight_integrity import IncompleteModelWeightsError, incomplete_weights_message from services.model_sources import ( missing_weight_variant, model_sources_are_downloaded, @@ -837,7 +838,15 @@ def get_ready_generator(self, model_id: str) -> BaseGenerator: pass else: gen._auto_download() - gen.load() + try: + gen.load() + except Exception as exc: + message = incomplete_weights_message( + manifest.get("name", gen.DISPLAY_NAME), exc + ) + if message is None: + raise + raise IncompleteModelWeightsError(message) from exc return gen def activate_ready_generator(self, model_id: str) -> BaseGenerator: diff --git a/api/services/weight_integrity.py b/api/services/weight_integrity.py new file mode 100644 index 00000000..2ab6b88b --- /dev/null +++ b/api/services/weight_integrity.py @@ -0,0 +1,38 @@ +"""Recognise a model load that failed because a weight file on disk is truncated. + +Older builds could rename a partially downloaded file into place, and the model +then looked installed while its weights could never load. The loader's own +error is a deep traceback; this turns it into something the user can act on. +""" +import re +from typing import Optional + +# Substrings that only appear when a weight file's bytes are cut short or garbled. +_TRUNCATED_WEIGHT_SIGNATURES = ( + # safetensors: header longer than the file, or a file shorter than its header says + "Error while deserializing header", + # torch.load on a cut-off .bin/.pt/.ckpt zip archive + "failed finding central directory", +) + +_CHECKPOINT_PATH = re.compile( + r"Unable to load weights from checkpoint file for '([^']+)'" +) + + +class IncompleteModelWeightsError(RuntimeError): + """A model's weight files on disk are incomplete; its message is user-facing.""" + + +def incomplete_weights_message(model_name: str, error: BaseException) -> Optional[str]: + """A short, actionable message when ``error`` comes from truncated weights, else None.""" + text = str(error) + if not any(signature in text for signature in _TRUNCATED_WEIGHT_SIGNATURES): + return None + match = _CHECKPOINT_PATH.search(text) + detail = f" ({match.group(1)})" if match else "" + return ( + f"The model files for {model_name} are incomplete or corrupted{detail}, " + "most likely from an interrupted download. Open the Models page, remove " + "this node's weights, then install them again." + ) diff --git a/api/tests/test_model_router.py b/api/tests/test_model_router.py index 4090a0ea..0a90f3f1 100644 --- a/api/tests/test_model_router.py +++ b/api/tests/test_model_router.py @@ -86,14 +86,15 @@ def tearDown(self) -> None: def install_hf_stub(self, files: dict[str, list[str]], calls: list[str]) -> None: module = types.ModuleType("huggingface_hub") - def list_repo_files(repo_id, revision=None, token=None): - calls.append(f"list:{repo_id}:{revision}:{token}") - return files[repo_id] + class HfApi: + def list_repo_tree(self, repo_id, revision=None, token=None, recursive=False): + calls.append(f"list:{repo_id}:{revision}:{token}") + return [types.SimpleNamespace(path=name, size=None) for name in files[repo_id]] def hf_hub_url(repo_id, filename, revision=None): return f"https://example.invalid/{repo_id}/{revision or 'main'}/{filename}" - module.list_repo_files = list_repo_files + module.HfApi = HfApi module.hf_hub_url = hf_hub_url sys.modules["huggingface_hub"] = module @@ -309,5 +310,119 @@ async def run(): self.assertEqual(switched, {"active": "demo/b"}) +class _FakeResponse: + """urlopen() stand-in serving ``body`` and, like http.client, returning b"" + at EOF even when Content-Length promised more.""" + + def __init__(self, body: bytes, content_length: int, status: int = 200) -> None: + self._body = body + self.status = status + self.headers = {"Content-Length": str(content_length)} + + def __enter__(self): + return self + + def __exit__(self, *exc) -> None: + return None + + def read(self, size: int) -> bytes: + chunk, self._body = self._body[:size], self._body[size:] + return chunk + + +class DownloadFileStreamedTests(unittest.TestCase): + FULL = b"0123456789" + + def setUp(self) -> None: + self.tempdir = tempfile.TemporaryDirectory(prefix="modly-download-") + self.dest = Path(self.tempdir.name) + self.control = {"pause": threading.Event(), "cancel": threading.Event()} + self.requests: list[dict] = [] + patches = [ + patch.object(model_router.time, "sleep", lambda _s: None), + patch.object(model_router, "_resolve_direct_download_url", lambda url, headers: url), + ] + for p in patches: + p.start() + self.addCleanup(p.stop) + + def tearDown(self) -> None: + self.tempdir.cleanup() + + def download(self, responses: list, expected_size=None) -> int: + def fake_urlopen(request, timeout=None): + self.requests.append(dict(request.header_items())) + return responses.pop(0) + + with patch.object(model_router, "urlopen", fake_urlopen): + return model_router._download_file_streamed( + url="https://example.invalid/w.safetensors", + filename="w.safetensors", + dest_dir=str(self.dest), + file_index=1, + total_files=1, + base_percent=1, + progress_cb=lambda _m: None, + control=self.control, + expected_size=expected_size, + ) + + def test_stream_closed_early_resumes_instead_of_saving_a_truncated_file(self) -> None: + size = self.download([ + _FakeResponse(self.FULL[:4], content_length=len(self.FULL)), + _FakeResponse(self.FULL[4:], content_length=6, status=206), + ]) + + self.assertEqual(size, len(self.FULL)) + self.assertEqual((self.dest / "w.safetensors").read_bytes(), self.FULL) + self.assertEqual(self.requests[1].get("Range"), "bytes=4-") + self.assertFalse((self.dest / "w.safetensors.part").exists()) + + def test_never_renames_a_short_file_into_place(self) -> None: + responses = [_FakeResponse(self.FULL[:4], content_length=len(self.FULL))] + responses += [_FakeResponse(b"", content_length=6, status=206) for _ in range(2)] + with self.assertRaisesRegex(RuntimeError, "received 4 of 10 bytes"): + self.download(responses) + + self.assertFalse((self.dest / "w.safetensors").exists()) + self.assertEqual((self.dest / "w.safetensors.part").read_bytes(), self.FULL[:4]) + + def test_existing_file_with_the_wrong_size_is_downloaded_again(self) -> None: + (self.dest / "w.safetensors").write_bytes(self.FULL[:4]) + size = self.download( + [_FakeResponse(self.FULL, content_length=len(self.FULL))], + expected_size=len(self.FULL), + ) + + self.assertEqual(size, len(self.FULL)) + self.assertEqual((self.dest / "w.safetensors").read_bytes(), self.FULL) + + def test_existing_file_with_the_expected_size_is_kept(self) -> None: + (self.dest / "w.safetensors").write_bytes(self.FULL) + self.assertEqual(self.download([], expected_size=len(self.FULL)), len(self.FULL)) + self.assertEqual(self.requests, []) + + def test_repo_listing_reports_file_sizes_and_skips_folders(self) -> None: + module = types.ModuleType("huggingface_hub") + seen: dict = {} + + class HfApi: + def list_repo_tree(self, repo_id, revision=None, token=None, recursive=False): + seen.update(repo_id=repo_id, revision=revision, token=token, recursive=recursive) + return [ + types.SimpleNamespace(path="transformer", tree_id="t"), + types.SimpleNamespace(path="transformer/w.safetensors", size=10), + ] + + module.HfApi = HfApi + with patch.dict(sys.modules, {"huggingface_hub": module}): + sizes = model_router._list_repo_file_sizes("org/repo", "main", "tok") + + self.assertEqual(sizes, {"transformer/w.safetensors": 10}) + self.assertEqual( + seen, {"repo_id": "org/repo", "revision": "main", "token": "tok", "recursive": True} + ) + + if __name__ == "__main__": unittest.main() diff --git a/api/tests/test_weight_integrity.py b/api/tests/test_weight_integrity.py new file mode 100644 index 00000000..792e56ab --- /dev/null +++ b/api/tests/test_weight_integrity.py @@ -0,0 +1,128 @@ +import asyncio +import tempfile +import threading +import unittest +from pathlib import Path + +import routers.generation as generation +import services.generator_registry as registry +from schemas.generation import JobStatus +from services.generator_registry import GeneratorRegistry +from services.weight_integrity import ( + IncompleteModelWeightsError, + incomplete_weights_message, +) + +# The tail of the traceback reported in issue #389 (TripoSG, diffusers). +TRUNCATED_SAFETENSORS = ( + "Traceback (most recent call last):\n" + "safetensors._safetensors_rust.SafetensorError: Error while deserializing header: " + "incomplete metadata, file not fully covered\n" + "OSError: Unable to load weights from checkpoint file for " + "'C:\\models\\triposg\\generate\\transformer\\diffusion_pytorch_model.safetensors' " + "at 'C:\\models\\triposg\\generate\\transformer\\diffusion_pytorch_model.safetensors'." +) + + +class _FailingLoadGenerator: + DISPLAY_NAME = "Fallback" + VRAM_GB = 1 + + def __init__(self, error: Exception) -> None: + self.error = error + + def is_downloaded(self) -> bool: + return True + + def is_loaded(self) -> bool: + return False + + def load(self) -> None: + raise self.error + + +class IncompleteWeightsMessageTests(unittest.TestCase): + def test_truncated_safetensors_names_the_file_and_the_fix(self) -> None: + message = incomplete_weights_message("TripoSG", RuntimeError(TRUNCATED_SAFETENSORS)) + self.assertIn("model files for TripoSG are incomplete", message) + self.assertIn("transformer\\diffusion_pytorch_model.safetensors", message) + self.assertIn("Models page", message) + self.assertNotIn("Traceback", message) + + def test_truncated_torch_archive_is_recognised(self) -> None: + error = RuntimeError( + "PytorchStreamReader failed reading zip archive: failed finding central directory" + ) + self.assertIsNotNone(incomplete_weights_message("Demo", error)) + + def test_unrelated_load_errors_are_left_alone(self) -> None: + self.assertIsNone( + incomplete_weights_message("Demo", RuntimeError("CUDA out of memory")) + ) + + +class RegistryLoadTests(unittest.TestCase): + def _registry(self, error: Exception) -> GeneratorRegistry: + reg = GeneratorRegistry() + reg._generators["triposg/generate"] = _FailingLoadGenerator(error) + reg._manifests["triposg/generate"] = {"name": "TripoSG"} + return reg + + def test_truncated_weights_raise_a_short_actionable_error(self) -> None: + reg = self._registry(RuntimeError(TRUNCATED_SAFETENSORS)) + with self.assertRaises(IncompleteModelWeightsError) as raised: + reg.get_ready_generator("triposg/generate") + self.assertIn("TripoSG", str(raised.exception)) + self.assertIsInstance(raised.exception.__cause__, RuntimeError) + + def test_other_load_errors_propagate_unchanged(self) -> None: + original = RuntimeError("CUDA out of memory") + reg = self._registry(original) + with self.assertRaises(RuntimeError) as raised: + reg.get_ready_generator("triposg/generate") + self.assertIs(raised.exception, original) + + +class GenerationJobErrorTests(unittest.TestCase): + def setUp(self) -> None: + self._prev_registry = generation.generator_registry + self._prev_ws = registry.WORKSPACE_DIR + self._tmp = tempfile.TemporaryDirectory() + registry.WORKSPACE_DIR = Path(self._tmp.name) + + def tearDown(self) -> None: + generation.generator_registry = self._prev_registry + registry.WORKSPACE_DIR = self._prev_ws + for store in ( + generation._jobs, + generation._cancel_events, + generation._cancelled, + generation._completed_at, + ): + store.clear() + self._tmp.cleanup() + + def test_job_error_is_the_short_message_not_the_traceback(self) -> None: + class _Registry: + def assert_weight_variant_installed(self, params, model_id=None) -> None: + pass + + def active_status(self) -> dict: + return {"loaded": True, "name": "TripoSG", "downloaded": True} + + def get_active(self): + raise IncompleteModelWeightsError("The model files for TripoSG are incomplete.") + + generation.generator_registry = _Registry() + job_id = "job-incomplete-weights" + generation._jobs[job_id] = JobStatus(job_id=job_id, status="pending", progress=0) + generation._cancel_events[job_id] = threading.Event() + asyncio.run(generation._run_generation(job_id, b"img", {}, "Coll")) + + job = generation._jobs[job_id] + self.assertEqual(job.status, "error") + self.assertEqual(job.error, "The model files for TripoSG are incomplete.") + + +if __name__ == "__main__": + unittest.main()