Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion api/routers/generation.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from services.generator_registry import generator_registry
from schemas.generation import GenerateFromArtifactRequest, JobStatus
from services.run_history import run_history
from services.weight_integrity import IncompleteModelWeightsError
from services.artifact_input import (
RESERVED_ARTIFACT_PARAMS,
TypedArtifactInput,
Expand Down Expand Up @@ -453,7 +454,8 @@ def progress_cb(pct: int, step: str = "") -> None:
except UnicodeEncodeError:
print(msg.encode("ascii", errors="replace").decode("ascii"))
job.status = "error"
job.error = tb.strip()
# Actionable errors reach the user as-is; the full traceback stays in the log.
job.error = str(exc) if isinstance(exc, IncompleteModelWeightsError) else tb.strip()
_completed_at[job_id] = time.monotonic()
finally:
_job_generators.pop(job_id, None)
71 changes: 56 additions & 15 deletions api/routers/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,26 +184,25 @@ def _fmt(data: dict) -> str:
try:
yield _fmt({"percent": 0, "status": "Listing repository files..."})
files_by_source: dict[str, list[str]] = {}
sizes_by_source: dict[str, dict[str, Optional[int]]] = {}

for source in sources:
_check_download_control(control)

def _list_files(current=source):
from huggingface_hub import list_repo_files
listed = list_repo_files(
current["repo_id"],
revision=current.get("revision"),
token=hf_token,
listed = _list_repo_file_sizes(
current["repo_id"], current.get("revision"), hf_token
)
include = current.get("include_prefixes", [])
skip = current.get("skip_prefixes", [])
return [
filename for filename in listed
return {
filename: size for filename, size in listed.items()
if (not include or any(filename.startswith(prefix) for prefix in include))
if not any(filename.startswith(prefix) for prefix in skip)
]
}

files = await loop.run_in_executor(None, _list_files)
listed_sizes = await loop.run_in_executor(None, _list_files)
files = list(listed_sizes)
if not files:
raise RuntimeError(
f'No files found in Hugging Face repo: {source["repo_id"]}'
Expand All @@ -212,6 +211,7 @@ def _list_files(current=source):
for filename in files:
resolve_download_path(destination, filename)
files_by_source[source["id"]] = files
sizes_by_source[source["id"]] = listed_sizes

validate_source_file_plan(sources, files_by_source)
planned_files = [
Expand Down Expand Up @@ -261,6 +261,7 @@ def _progress(message: dict) -> None:
progress_cb=_progress,
control=control,
token=hf_token,
expected_size=sizes_by_source[source["id"]].get(filename),
),
)
while not future.done():
Expand Down Expand Up @@ -364,14 +365,14 @@ def _fmt(data: dict) -> str:
_check_download_control(control)

def _list_files():
from huggingface_hub import list_repo_files
return [
f for f in list_repo_files(repo_id, token=hf_token)
return {
f: size for f, size in _list_repo_file_sizes(repo_id, None, hf_token).items()
if (not include_list or any(f.startswith(p) for p in include_list))
if not any(f.startswith(p) for p in skip_list)
]
}

files = await loop.run_in_executor(None, _list_files)
file_sizes = await loop.run_in_executor(None, _list_files)
files = list(file_sizes)
total = len(files)

if total == 0:
Expand Down Expand Up @@ -413,6 +414,7 @@ def _progress(msg: dict) -> None:
progress_cb=_progress,
control=control,
token=hf_token,
expected_size=file_sizes.get(filename),
),
)

Expand Down Expand Up @@ -459,6 +461,30 @@ def _progress(msg: dict) -> None:
return StreamingResponse(stream(), media_type="text/event-stream")


class IncompleteDownload(OSError):
"""The server closed the stream before the whole file arrived.
An OSError so the retry loop resumes from the .part file. http.client does
not raise on a short body (read() returns b"" at EOF), so without this check
a truncated download was renamed into place and later treated as complete.
"""


def _list_repo_file_sizes(
repo_id: str, revision: Optional[str], token: Optional[str]
) -> dict[str, Optional[int]]:
"""Every file of a Hugging Face repo with its size in bytes, in one API call."""
from huggingface_hub import HfApi
return {
entry.path: entry.size
for entry in HfApi().list_repo_tree(
repo_id, revision=revision, token=token, recursive=True
)
# RepoFolder entries have no size; RepoFile entries always do.
if hasattr(entry, "size")
}


def _download_file_streamed(
*,
url: str,
Expand All @@ -470,6 +496,7 @@ def _download_file_streamed(
progress_cb,
control: dict[str, threading.Event],
token: Optional[str] = None,
expected_size: Optional[int] = None,
) -> int:
final_path = Path(dest_dir) / filename
temp_path = final_path.with_suffix(final_path.suffix + ".part")
Expand All @@ -479,8 +506,9 @@ def _download_file_streamed(
if not final_path.is_file():
raise RuntimeError(f"Download target is not a regular file: {filename}")
existing_size = final_path.stat().st_size
if existing_size > 0:
if existing_size > 0 and (expected_size is None or existing_size == expected_size):
return existing_size
# Empty, or truncated by an older build: fetch it again.
final_path.unlink()

# Explicit token (from caller) > env vars > none
Expand All @@ -502,6 +530,12 @@ def _download_file_streamed(
try:
_check_download_control(control)
existing_bytes = temp_path.stat().st_size if temp_path.exists() else 0
if expected_size is not None and existing_bytes > expected_size:
temp_path.unlink()
existing_bytes = 0
if expected_size is not None and existing_bytes == expected_size and existing_bytes > 0:
temp_path.replace(final_path)
return existing_bytes
request_headers = dict(headers)
request_url = url
if existing_bytes > 0:
Expand Down Expand Up @@ -560,6 +594,13 @@ def _download_file_streamed(
})
last_emit = now

expected_bytes = expected_size if expected_size is not None else total_bytes
if expected_bytes is not None and bytes_downloaded != expected_bytes:
if bytes_downloaded > expected_bytes:
temp_path.unlink(missing_ok=True)
raise IncompleteDownload(
f"{filename}: received {bytes_downloaded} of {expected_bytes} bytes"
)
temp_path.replace(final_path)
return bytes_downloaded

Expand Down
11 changes: 10 additions & 1 deletion api/services/generator_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

from services.generators.base import BaseGenerator
from services.extension_process import ExtensionProcess, _venv_python
from services.weight_integrity import IncompleteModelWeightsError, incomplete_weights_message
from services.model_sources import (
missing_weight_variant,
model_sources_are_downloaded,
Expand Down Expand Up @@ -837,7 +838,15 @@ def get_ready_generator(self, model_id: str) -> BaseGenerator:
pass
else:
gen._auto_download()
gen.load()
try:
gen.load()
except Exception as exc:
message = incomplete_weights_message(
manifest.get("name", gen.DISPLAY_NAME), exc
)
if message is None:
raise
raise IncompleteModelWeightsError(message) from exc
return gen

def activate_ready_generator(self, model_id: str) -> BaseGenerator:
Expand Down
38 changes: 38 additions & 0 deletions api/services/weight_integrity.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
"""Recognise a model load that failed because a weight file on disk is truncated.

Older builds could rename a partially downloaded file into place, and the model
then looked installed while its weights could never load. The loader's own
error is a deep traceback; this turns it into something the user can act on.
"""
import re
from typing import Optional

# Substrings that only appear when a weight file's bytes are cut short or garbled.
_TRUNCATED_WEIGHT_SIGNATURES = (
# safetensors: header longer than the file, or a file shorter than its header says
"Error while deserializing header",
# torch.load on a cut-off .bin/.pt/.ckpt zip archive
"failed finding central directory",
)

_CHECKPOINT_PATH = re.compile(
r"Unable to load weights from checkpoint file for '([^']+)'"
)


class IncompleteModelWeightsError(RuntimeError):
"""A model's weight files on disk are incomplete; its message is user-facing."""


def incomplete_weights_message(model_name: str, error: BaseException) -> Optional[str]:
"""A short, actionable message when ``error`` comes from truncated weights, else None."""
text = str(error)
if not any(signature in text for signature in _TRUNCATED_WEIGHT_SIGNATURES):
return None
match = _CHECKPOINT_PATH.search(text)
detail = f" ({match.group(1)})" if match else ""
return (
f"The model files for {model_name} are incomplete or corrupted{detail}, "
"most likely from an interrupted download. Open the Models page, remove "
"this node's weights, then install them again."
)
123 changes: 119 additions & 4 deletions api/tests/test_model_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,14 +86,15 @@ def tearDown(self) -> None:
def install_hf_stub(self, files: dict[str, list[str]], calls: list[str]) -> None:
module = types.ModuleType("huggingface_hub")

def list_repo_files(repo_id, revision=None, token=None):
calls.append(f"list:{repo_id}:{revision}:{token}")
return files[repo_id]
class HfApi:
def list_repo_tree(self, repo_id, revision=None, token=None, recursive=False):
calls.append(f"list:{repo_id}:{revision}:{token}")
return [types.SimpleNamespace(path=name, size=None) for name in files[repo_id]]

def hf_hub_url(repo_id, filename, revision=None):
return f"https://example.invalid/{repo_id}/{revision or 'main'}/{filename}"

module.list_repo_files = list_repo_files
module.HfApi = HfApi
module.hf_hub_url = hf_hub_url
sys.modules["huggingface_hub"] = module

Expand Down Expand Up @@ -309,5 +310,119 @@ async def run():
self.assertEqual(switched, {"active": "demo/b"})


class _FakeResponse:
"""urlopen() stand-in serving ``body`` and, like http.client, returning b""
at EOF even when Content-Length promised more."""

def __init__(self, body: bytes, content_length: int, status: int = 200) -> None:
self._body = body
self.status = status
self.headers = {"Content-Length": str(content_length)}

def __enter__(self):
return self

def __exit__(self, *exc) -> None:
return None

def read(self, size: int) -> bytes:
chunk, self._body = self._body[:size], self._body[size:]
return chunk


class DownloadFileStreamedTests(unittest.TestCase):
FULL = b"0123456789"

def setUp(self) -> None:
self.tempdir = tempfile.TemporaryDirectory(prefix="modly-download-")
self.dest = Path(self.tempdir.name)
self.control = {"pause": threading.Event(), "cancel": threading.Event()}
self.requests: list[dict] = []
patches = [
patch.object(model_router.time, "sleep", lambda _s: None),
patch.object(model_router, "_resolve_direct_download_url", lambda url, headers: url),
]
for p in patches:
p.start()
self.addCleanup(p.stop)

def tearDown(self) -> None:
self.tempdir.cleanup()

def download(self, responses: list, expected_size=None) -> int:
def fake_urlopen(request, timeout=None):
self.requests.append(dict(request.header_items()))
return responses.pop(0)

with patch.object(model_router, "urlopen", fake_urlopen):
return model_router._download_file_streamed(
url="https://example.invalid/w.safetensors",
filename="w.safetensors",
dest_dir=str(self.dest),
file_index=1,
total_files=1,
base_percent=1,
progress_cb=lambda _m: None,
control=self.control,
expected_size=expected_size,
)

def test_stream_closed_early_resumes_instead_of_saving_a_truncated_file(self) -> None:
size = self.download([
_FakeResponse(self.FULL[:4], content_length=len(self.FULL)),
_FakeResponse(self.FULL[4:], content_length=6, status=206),
])

self.assertEqual(size, len(self.FULL))
self.assertEqual((self.dest / "w.safetensors").read_bytes(), self.FULL)
self.assertEqual(self.requests[1].get("Range"), "bytes=4-")
self.assertFalse((self.dest / "w.safetensors.part").exists())

def test_never_renames_a_short_file_into_place(self) -> None:
responses = [_FakeResponse(self.FULL[:4], content_length=len(self.FULL))]
responses += [_FakeResponse(b"", content_length=6, status=206) for _ in range(2)]
with self.assertRaisesRegex(RuntimeError, "received 4 of 10 bytes"):
self.download(responses)

self.assertFalse((self.dest / "w.safetensors").exists())
self.assertEqual((self.dest / "w.safetensors.part").read_bytes(), self.FULL[:4])

def test_existing_file_with_the_wrong_size_is_downloaded_again(self) -> None:
(self.dest / "w.safetensors").write_bytes(self.FULL[:4])
size = self.download(
[_FakeResponse(self.FULL, content_length=len(self.FULL))],
expected_size=len(self.FULL),
)

self.assertEqual(size, len(self.FULL))
self.assertEqual((self.dest / "w.safetensors").read_bytes(), self.FULL)

def test_existing_file_with_the_expected_size_is_kept(self) -> None:
(self.dest / "w.safetensors").write_bytes(self.FULL)
self.assertEqual(self.download([], expected_size=len(self.FULL)), len(self.FULL))
self.assertEqual(self.requests, [])

def test_repo_listing_reports_file_sizes_and_skips_folders(self) -> None:
module = types.ModuleType("huggingface_hub")
seen: dict = {}

class HfApi:
def list_repo_tree(self, repo_id, revision=None, token=None, recursive=False):
seen.update(repo_id=repo_id, revision=revision, token=token, recursive=recursive)
return [
types.SimpleNamespace(path="transformer", tree_id="t"),
types.SimpleNamespace(path="transformer/w.safetensors", size=10),
]

module.HfApi = HfApi
with patch.dict(sys.modules, {"huggingface_hub": module}):
sizes = model_router._list_repo_file_sizes("org/repo", "main", "tok")

self.assertEqual(sizes, {"transformer/w.safetensors": 10})
self.assertEqual(
seen, {"repo_id": "org/repo", "revision": "main", "token": "tok", "recursive": True}
)


if __name__ == "__main__":
unittest.main()
Loading
Loading