Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 24 additions & 3 deletions api/routers/generation.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,29 @@ def record_job_filed(
inputs: dict,
params: dict,
collection: str,
parent_run_id: Optional[str] = None,
) -> None:
"""File the persisted history record of a generation job (see services.run_history)."""
run_history.begin(
job_id, "generation", model_id=model_id, inputs=inputs,
params={**params, "collection": collection},
params={**params, "collection": collection}, parent_run_id=parent_run_id,
)


MAX_RUN_ID_CHARS = 200


def normalize_parent_run_id(value: object) -> Optional[str]:
"""The optional ``parent_run_id`` a caller sent: the workflow run that launched this
generation. Blank means none (a multipart form sends "" rather than omitting it)."""
if not isinstance(value, str) or not value.strip():
return None
value = value.strip()
if len(value) > MAX_RUN_ID_CHARS:
raise HTTPException(400, "parent_run_id is too long")
return value


def _record_job_end(job_id: str) -> None:
"""Copy a job's final state into its history record once its worker returned."""
job = _jobs.get(job_id)
Expand Down Expand Up @@ -134,9 +149,11 @@ async def generate_from_image(
enable_texture: bool = Form(False),
texture_resolution: int = Form(1024),
params: str = Form("{}"),
parent_run_id: Optional[str] = Form(None),
):
if not image.content_type or not image.content_type.startswith("image/"):
raise HTTPException(400, "File must be an image")
parent_run_id = normalize_parent_run_id(parent_run_id)

if remesh not in VALID_REMESH_MODES:
raise HTTPException(400, "remesh must be 'quad', 'triangle', or 'none'")
Expand Down Expand Up @@ -171,7 +188,9 @@ async def generate_from_image(
_jobs[job_id] = job
_cancel_events[job_id] = threading.Event()

record_job_filed(job_id, model_id, {"image": image.filename}, full_params, collection)
record_job_filed(
job_id, model_id, {"image": image.filename}, full_params, collection, parent_run_id
)

background_tasks.add_task(
_run_generation, job_id, image_bytes, full_params, collection, output_kind, model_id
Expand All @@ -186,6 +205,7 @@ async def generate_from_artifact(
background_tasks: BackgroundTasks,
):
"""Queue a validated typed artifact without serializing it as image bytes."""
parent_run_id = normalize_parent_run_id(request.parent_run_id)
try:
manifest = generator_registry.get_manifest(request.model_id)
except (KeyError, ValueError) as exc:
Expand All @@ -202,7 +222,8 @@ async def generate_from_artifact(
collection = sanitize_collection(request.collection)
job_id = str(uuid.uuid4())
record_job_filed(
job_id, request.model_id, {request.input_kind: request.input_path}, params, collection
job_id, request.model_id, {request.input_kind: request.input_path}, params, collection,
parent_run_id,
)
params["scene_manifest_path"] = str(artifact.path)
_purge_old_jobs()
Expand Down
19 changes: 17 additions & 2 deletions api/routers/workflow_runs.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ class WorkflowRunRecordIn(BaseModel):
ended_at: Optional[float] = None


class RunRecordChildren(BaseModel):
runs: list[RunRecord]


MAX_PAGE = 200


Expand All @@ -58,12 +62,23 @@ async def list_runs(
kind: Optional[list[RunKind]] = Query(None),
limit: int = Query(50, ge=1, le=MAX_PAGE),
offset: int = Query(0, ge=0),
top_level: bool = False,
):
"""Persisted run history, newest first. ``status``/``kind`` may repeat to match any."""
runs, total = run_history.list(statuses=status, kinds=kind, limit=limit, offset=offset)
"""Persisted run history, newest first. ``status``/``kind`` may repeat to match any.
``top_level`` leaves out runs launched by another run (a workflow's generations);
fetch those with ``/{run_id}/children``. A running record carries its live steps."""
runs, total = run_history.list(
statuses=status, kinds=kind, limit=limit, offset=offset, top_level=top_level
)
return RunRecordPage(runs=runs, total=total, limit=limit, offset=offset)


@router.get("/{run_id}/children", response_model=RunRecordChildren)
async def list_run_children(run_id: str):
"""The runs a run launched (a workflow's generations), in execution order."""
return RunRecordChildren(runs=run_history.children(run_id))


@router.put("/{run_id}", response_model=RunRecord)
async def put_workflow_run_record(run_id: str, body: WorkflowRunRecordIn):
"""Insert or replace the history record of a renderer-driven workflow run."""
Expand Down
1 change: 1 addition & 0 deletions api/schemas/generation.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,4 @@ class GenerateFromArtifactRequest(BaseModel):
model_id: str
collection: str = "Workflows"
params: dict[str, Any] = Field(default_factory=dict)
parent_run_id: Optional[str] = None # the workflow run this generation belongs to
3 changes: 3 additions & 0 deletions api/schemas/runs.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ class RunRecord(BaseModel):
workflow_id: Optional[str] = None
workflow_name: Optional[str] = None
op_id: Optional[str] = None # mesh operation id, e.g. "decimate"
# The run this one was launched by: a workflow run's model nodes each file their own
# generation, linked back here so a history feed can fold them under the workflow.
parent_run_id: Optional[str] = None
inputs: dict[str, Any] = Field(default_factory=dict)
params: dict[str, Any] = Field(default_factory=dict)
steps: list[RunStep] = Field(default_factory=list)
Expand Down
22 changes: 19 additions & 3 deletions api/services/run_history.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,13 +113,14 @@ def begin(
op_id: Optional[str] = None,
inputs: Optional[dict[str, Any]] = None,
params: Optional[dict[str, Any]] = None,
parent_run_id: Optional[str] = None,
) -> None:
"""File a pending run (persisted, so a crash before it starts still shows)."""
if not self.enabled:
return
record = RunRecord(
id=run_id, kind=kind, status="pending", model_id=model_id, op_id=op_id,
inputs=inputs or {}, params=params or {},
inputs=inputs or {}, params=params or {}, parent_run_id=parent_run_id,
created_at=time.time(),
)
self.save(record)
Expand All @@ -135,7 +136,9 @@ def mark_running(self, run_id: str) -> None:

def step(self, run_id: str, name: str) -> None:
"""Note the current step label. In memory only -- labels can change many times
a second; the steps are written out with the next status change."""
a second; the steps are written out with the next status change. Readers still
see them live: a live record is the very object ``_records`` holds, and
``get``/``list`` copy it at read time."""
if not name:
return
with self._lock:
Expand Down Expand Up @@ -214,8 +217,10 @@ def list(
kinds: Optional[Iterable[str]] = None,
limit: int = 50,
offset: int = 0,
top_level: bool = False,
) -> tuple[list[RunRecord], int]:
"""Newest first. Returns (page, total matching)."""
"""Newest first. Returns (page, total matching). ``top_level`` leaves out runs
launched by another run (see ``children``)."""
if not self.enabled:
return [], 0
wanted_status = set(statuses) if statuses else None
Expand All @@ -226,11 +231,22 @@ def list(
r for r in self._records.values()
if (wanted_status is None or r.status in wanted_status)
and (wanted_kind is None or r.kind in wanted_kind)
and not (top_level and r.parent_run_id)
]
matching.sort(key=lambda r: r.created_at, reverse=True)
page = matching[offset:offset + limit]
return [r.model_copy(deep=True) for r in page], len(matching)

def children(self, parent_run_id: str) -> list[RunRecord]:
"""The runs launched by ``parent_run_id``, in execution order (oldest first)."""
if not self.enabled:
return []
with self._lock:
self._sync_path()
matching = [r for r in self._records.values() if r.parent_run_id == parent_run_id]
matching.sort(key=lambda r: r.created_at)
return [r.model_copy(deep=True) for r in matching]

# ------------------------------------------------------------------ #
# Internals (lock held)
# ------------------------------------------------------------------ #
Expand Down
135 changes: 133 additions & 2 deletions api/tests/test_run_history.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@
from pathlib import Path
from unittest.mock import patch

from fastapi import HTTPException
from fastapi import BackgroundTasks, HTTPException

import routers.generation as generation
import routers.optimize as optimize
import routers.workflow_runs as workflow_runs
import services.run_history as run_history_module
from schemas.generation import JobStatus
from schemas.generation import GenerateFromArtifactRequest, JobStatus
from schemas.runs import RunRecord, RunStep
from services.mesh_ops import MeshOpExecutionError, MeshOpResult
from services.run_history import (
Expand Down Expand Up @@ -190,6 +190,57 @@ def test_follows_a_relocated_workspace(self) -> None:
self.assertEqual(reloaded.get("live").status, "done")


class ParentRunTests(_TempStoreCase):
def _child(self, run_id: str, parent: str, created_at: float) -> RunRecord:
record = _record(run_id, created_at=created_at)
record.parent_run_id = parent
return record

def test_parent_run_id_is_persisted(self) -> None:
self.store.begin("child", "generation", model_id="sf3d", parent_run_id="wf-run")
self.assertEqual(self.reopen().get("child").parent_run_id, "wf-run")

def test_old_history_without_parent_run_id_loads(self) -> None:
self.path.write_text(json.dumps({"version": 1, "runs": [
{"id": "legacy", "kind": "generation", "status": "done", "created_at": 1000.0},
]}), encoding="utf-8")
record = self.reopen().get("legacy")
self.assertEqual(record.status, "done")
self.assertIsNone(record.parent_run_id)

def test_top_level_leaves_out_children_and_counts_accordingly(self) -> None:
self.store.save(_record("wf", kind="workflow", created_at=1000.0))
self.store.save(self._child("c1", "wf", 1001.0))
self.store.save(self._child("c2", "wf", 1002.0))
self.store.save(_record("solo", created_at=1003.0))
page, total = self.store.list(top_level=True)
self.assertEqual(([r.id for r in page], total), (["solo", "wf"], 2))
page, total = self.store.list()
self.assertEqual(total, 4)

def test_children_are_in_execution_order(self) -> None:
self.store.save(self._child("late", "wf", 1005.0))
self.store.save(self._child("early", "wf", 1001.0))
self.store.save(self._child("other", "wf-2", 1002.0))
self.store.save(_record("wf", kind="workflow"))
self.assertEqual([r.id for r in self.store.children("wf")], ["early", "late"])
self.assertEqual(self.store.children("nobody"), [])

def test_running_record_lists_its_live_steps_without_rewriting_the_file(self) -> None:
self.store.begin("live", "generation")
self.store.mark_running("live")
on_disk = self.path.read_text(encoding="utf-8")
self.store.step("live", "Loading")
self.store.step("live", "Sampling 3/50")
page, _ = self.store.list(statuses=["running"])
self.assertEqual([s.name for s in page[0].steps], ["Loading", "Sampling 3/50"])
self.assertIsNone(page[0].steps[-1].ended_at)
self.assertEqual(self.path.read_text(encoding="utf-8"), on_disk)
# The list hands out copies: a caller cannot corrupt the live record.
page[0].steps.clear()
self.assertEqual(len(self.store.get("live").steps), 2)


class _RouterHistoryCase(_TempStoreCase):
"""Points the routers' shared store at a temp workspace for one test."""

Expand Down Expand Up @@ -221,6 +272,29 @@ def test_route_is_mounted_at_the_collection_root(self) -> None:
paths = {(route.path, tuple(sorted(route.methods))) for route in workflow_runs.router.routes}
self.assertIn(("", ("GET",)), paths)
self.assertIn(("/{run_id}", ("PUT",)), paths)
self.assertIn(("/{run_id}/children", ("GET",)), paths)

def test_top_level_filter_and_children_route(self) -> None:
self.shared.save(_record("wf", kind="workflow", created_at=1000.0))
for i in range(2):
child = _record(f"c{i}", created_at=1001.0 + i)
child.parent_run_id = "wf"
self.shared.save(child)
page = asyncio.run(workflow_runs.list_runs(status=None, kind=None, limit=50, offset=0, top_level=True))
self.assertEqual(([r.id for r in page.runs], page.total), (["wf"], 1))
page = asyncio.run(workflow_runs.list_runs(status=None, kind=None, limit=50, offset=0))
self.assertEqual(page.total, 3)
self.assertEqual(page.runs[0].parent_run_id, "wf")
children = asyncio.run(workflow_runs.list_run_children("wf"))
self.assertEqual([r.id for r in children.runs], ["c0", "c1"])
self.assertEqual(asyncio.run(workflow_runs.list_run_children("c0")).runs, [])

def test_children_path_does_not_resolve_to_get_run(self) -> None:
from starlette.routing import Match
scope = {"type": "http", "method": "GET", "path": "/wf/children"}
matched = [r.endpoint for r in workflow_runs.router.routes
if r.matches(scope)[0] == Match.FULL]
self.assertEqual(matched, [workflow_runs.list_run_children])

def test_get_run_falls_back_to_history_after_purge(self) -> None:
record = _record("old", "done")
Expand Down Expand Up @@ -331,6 +405,63 @@ def test_cancel_is_recorded_immediately(self) -> None:
self.assertEqual(self.reopen().get("c").status, "cancelled")


class _FakeUpload:
filename = "in.png"
content_type = "image/png"

async def read(self) -> bytes:
return b"img"


class _EndpointRegistry:
def get_generator(self, model_id):
return object()

def get_manifest(self, model_id):
return {"output": "mesh", "inputs": ["image", "scene"]}


class GenerationParentRunTests(_RouterHistoryCase):
def setUp(self) -> None:
super().setUp()
for patcher in (patch.object(generation.registry, "WORKSPACE_DIR", self.workspace),
patch.object(generation, "generator_registry", _EndpointRegistry())):
patcher.start()
self.addCleanup(patcher.stop)

def _from_image(self, **extra) -> str:
response = asyncio.run(generation.generate_from_image(
BackgroundTasks(), image=_FakeUpload(), model_id="demo/model", collection="Workflows",
remesh="none", enable_texture=False, texture_resolution=1024, params="{}", **extra,
))
return response["job_id"]

def test_from_image_records_the_parent_run(self) -> None:
job_id = self._from_image(parent_run_id="wf-run")
self.assertEqual(self.reopen().get(job_id).parent_run_id, "wf-run")
self.assertEqual([r.id for r in self.shared.children("wf-run")], [job_id])

def test_from_image_without_or_with_blank_parent_is_top_level(self) -> None:
self.assertIsNone(self.shared.get(self._from_image()).parent_run_id)
self.assertIsNone(self.shared.get(self._from_image(parent_run_id=" ")).parent_run_id)

def test_from_image_refuses_an_oversized_parent_id(self) -> None:
with self.assertRaises(HTTPException) as ctx:
self._from_image(parent_run_id="x" * 201)
self.assertEqual(ctx.exception.status_code, 400)

def test_from_artifact_records_the_parent_run(self) -> None:
scene = self.workspace / "Workflows" / "room"
scene.mkdir(parents=True)
(scene / "scene-manifest.json").write_text(
json.dumps({"schema": "modly.scene-manifest.v1", "sceneRoot": ".", "assets": []}))
response = asyncio.run(generation.generate_from_artifact(GenerateFromArtifactRequest(
input_kind="scene", input_path="Workflows/room", model_id="demo/scene",
parent_run_id="wf-run",
), BackgroundTasks()))
self.assertEqual(self.reopen().get(response["job_id"]).parent_run_id, "wf-run")


class MeshOpRecordingTests(_RouterHistoryCase):
def setUp(self) -> None:
super().setUp()
Expand Down
6 changes: 6 additions & 0 deletions electron/main/ipc-handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ import {
import { registerWorkspaceAssetLibraryIpcHandlers } from './artifact-registry-service'
import { getThumbnailsDir } from './asset-thumbnail-protocol'
import { registerAssetThumbnailIpcHandlers, removeThumbnails } from './asset-thumbnail-cache'
import { registerWorkspaceImageIpcHandlers } from './workspace-images'
import { updatesSupported } from './updater'
import { ModelWeightOperations } from './model-weight-operations'
import { readLocalFileBase64 } from './bounded-file-reader'
Expand Down Expand Up @@ -1011,6 +1012,11 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe
getWorkspaceDir: () => getSettings(app.getPath('userData')).workspaceDir,
thumbnailsDir: getThumbnailsDir(),
})
registerWorkspaceImageIpcHandlers({
ipcMain,
getWorkspaceDir: () => getSettings(app.getPath('userData')).workspaceDir,
showItemInFolder: (path) => shell.showItemInFolder(path),
})

ipcMain.handle('workspace:listCollections', async () => {
const base = workspacePath()
Expand Down
Loading
Loading