fix: reserve and persist background jobs

dashboard/jobs.py

@@ -0,0 +1,111 @@
+"""Persistent, thread-safe background job state."""
+
+from __future__ import annotations
+
+import threading
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Callable
+
+from dashboard.persistence import atomic_write_json, load_json
+
+_ACTIVE = {"queued", "running"}
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+class JobRegistry:
+    """Reserve jobs before spawning and persist their lifecycle."""
+
+    def __init__(self, path: str | Path, *, history_limit: int = 100):
+        self.path = Path(path)
+        self.history_limit = history_limit
+        self._lock = threading.RLock()
+        loaded = load_json(self.path, {"jobs": []}) or {"jobs": []}
+        self._jobs = {
+            job["id"]: dict(job)
+            for job in loaded.get("jobs", [])
+            if isinstance(job, dict) and job.get("id")
+        }
+        changed = False
+        for job in self._jobs.values():
+            if job.get("status") in _ACTIVE:
+                job.update(
+                    status="interrupted",
+                    finished_at=_now(),
+                    error="process restarted before job completed",
+                )
+                changed = True
+        if changed:
+            self._save_locked()
+
+    def _save_locked(self) -> None:
+        jobs = sorted(self._jobs.values(), key=lambda job: job.get("created_at", ""))
+        if len(jobs) > self.history_limit:
+            keep = jobs[-self.history_limit :]
+            self._jobs = {job["id"]: job for job in keep}
+            jobs = keep
+        atomic_write_json(self.path, {"jobs": jobs})
+
+    def reserve(self, kind: str, *, details: dict[str, Any] | None = None) -> dict[str, Any] | None:
+        with self._lock:
+            if any(
+                job.get("kind") == kind and job.get("status") in _ACTIVE
+                for job in self._jobs.values()
+            ):
+                return None
+            job = {
+                "id": uuid.uuid4().hex,
+                "kind": kind,
+                "status": "queued",
+                "created_at": _now(),
+                "started_at": None,
+                "finished_at": None,
+                "progress": {},
+                "details": details or {},
+                "result": None,
+                "error": None,
+            }
+            self._jobs[job["id"]] = job
+            self._save_locked()
+            return dict(job)
+
+    def get(self, job_id: str) -> dict[str, Any] | None:
+        with self._lock:
+            job = self._jobs.get(job_id)
+            return dict(job) if job else None
+
+    def active(self, kind: str) -> dict[str, Any] | None:
+        with self._lock:
+            for job in self._jobs.values():
+                if job.get("kind") == kind and job.get("status") in _ACTIVE:
+                    return dict(job)
+        return None
+
+    def update_progress(self, job_id: str, progress: dict[str, Any]) -> None:
+        with self._lock:
+            job = self._jobs[job_id]
+            job["progress"] = dict(progress)
+            self._save_locked()
+
+    def run(self, job_id: str, operation: Callable[[], Any]) -> Any:
+        with self._lock:
+            job = self._jobs[job_id]
+            if job["status"] != "queued":
+                raise RuntimeError(f"job {job_id} is not queued")
+            job.update(status="running", started_at=_now())
+            self._save_locked()
+        try:
+            result = operation()
+        except Exception as exc:
+            with self._lock:
+                job.update(status="error", error=str(exc), finished_at=_now())
+                self._save_locked()
+            raise
+        with self._lock:
+            job.update(status="complete", result=result, finished_at=_now())
+            self._save_locked()
+        return result