112 lines
3.7 KiB
Python
112 lines
3.7 KiB
Python
"""Persistent, thread-safe background job state."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
from dashboard.persistence import atomic_write_json, load_json
|
|
|
|
_ACTIVE = {"queued", "running"}
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
class JobRegistry:
|
|
"""Reserve jobs before spawning and persist their lifecycle."""
|
|
|
|
def __init__(self, path: str | Path, *, history_limit: int = 100):
|
|
self.path = Path(path)
|
|
self.history_limit = history_limit
|
|
self._lock = threading.RLock()
|
|
loaded = load_json(self.path, {"jobs": []}) or {"jobs": []}
|
|
self._jobs = {
|
|
job["id"]: dict(job)
|
|
for job in loaded.get("jobs", [])
|
|
if isinstance(job, dict) and job.get("id")
|
|
}
|
|
changed = False
|
|
for job in self._jobs.values():
|
|
if job.get("status") in _ACTIVE:
|
|
job.update(
|
|
status="interrupted",
|
|
finished_at=_now(),
|
|
error="process restarted before job completed",
|
|
)
|
|
changed = True
|
|
if changed:
|
|
self._save_locked()
|
|
|
|
def _save_locked(self) -> None:
|
|
jobs = sorted(self._jobs.values(), key=lambda job: job.get("created_at", ""))
|
|
if len(jobs) > self.history_limit:
|
|
keep = jobs[-self.history_limit :]
|
|
self._jobs = {job["id"]: job for job in keep}
|
|
jobs = keep
|
|
atomic_write_json(self.path, {"jobs": jobs})
|
|
|
|
def reserve(self, kind: str, *, details: dict[str, Any] | None = None) -> dict[str, Any] | None:
|
|
with self._lock:
|
|
if any(
|
|
job.get("kind") == kind and job.get("status") in _ACTIVE
|
|
for job in self._jobs.values()
|
|
):
|
|
return None
|
|
job = {
|
|
"id": uuid.uuid4().hex,
|
|
"kind": kind,
|
|
"status": "queued",
|
|
"created_at": _now(),
|
|
"started_at": None,
|
|
"finished_at": None,
|
|
"progress": {},
|
|
"details": details or {},
|
|
"result": None,
|
|
"error": None,
|
|
}
|
|
self._jobs[job["id"]] = job
|
|
self._save_locked()
|
|
return dict(job)
|
|
|
|
def get(self, job_id: str) -> dict[str, Any] | None:
|
|
with self._lock:
|
|
job = self._jobs.get(job_id)
|
|
return dict(job) if job else None
|
|
|
|
def active(self, kind: str) -> dict[str, Any] | None:
|
|
with self._lock:
|
|
for job in self._jobs.values():
|
|
if job.get("kind") == kind and job.get("status") in _ACTIVE:
|
|
return dict(job)
|
|
return None
|
|
|
|
def update_progress(self, job_id: str, progress: dict[str, Any]) -> None:
|
|
with self._lock:
|
|
job = self._jobs[job_id]
|
|
job["progress"] = dict(progress)
|
|
self._save_locked()
|
|
|
|
def run(self, job_id: str, operation: Callable[[], Any]) -> Any:
|
|
with self._lock:
|
|
job = self._jobs[job_id]
|
|
if job["status"] != "queued":
|
|
raise RuntimeError(f"job {job_id} is not queued")
|
|
job.update(status="running", started_at=_now())
|
|
self._save_locked()
|
|
try:
|
|
result = operation()
|
|
except Exception as exc:
|
|
with self._lock:
|
|
job.update(status="error", error=str(exc), finished_at=_now())
|
|
self._save_locked()
|
|
raise
|
|
with self._lock:
|
|
job.update(status="complete", result=result, finished_at=_now())
|
|
self._save_locked()
|
|
return result
|