fix: reject unprovenanced ML artifacts

backtesting/engine.py

@@ -8,6 +8,7 @@ from collections import defaultdict
 from datetime import datetime, timedelta
 
 from scoring.policy import SCORE_BRACKETS, SCORE_VERSION, score_in_bracket
+from ml.artifacts import validate_ml_artifact
 
 log = logging.getLogger(__name__)
 
@@ -117,11 +118,15 @@ def _compute_ath_series(price_lookup, dates):
 
 
 def _load_ml_weights():
-    """Load ML weights for ML-optimized scoring mode."""
+    """Load only schema/provenance-valid ML weights."""
     ml_path = _os.path.join(_os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))), "config", "ml_weights.json")
     try:
         with open(ml_path) as f:
             data = _json.load(f)
+        status = validate_ml_artifact(data)
+        if not status["valid"]:
+            log.error("Rejected invalid ML artifact: %s", ", ".join(status["errors"]))
+            return {}
         return data.get("weights", {})
     except Exception:
         return {}

ml/artifacts.py

@@ -0,0 +1,91 @@
+"""Validation for persisted ML scoring artifacts."""
+
+import math
+
+from scoring.policy import SCORE_VERSION
+
+ML_ARTIFACT_SCHEMA_VERSION = 2
+REQUIRED_WEIGHT_KEYS = frozenset({
+    "puell_multiple",
+    "mvrv_zscore",
+    "reserve_risk",
+    "rhodl_ratio",
+    "nupl",
+    "fear_greed",
+    "drawdown",
+    "pct_above_200w_sma",
+    "pct_above_lth_rp",
+})
+
+
+def _weights_valid(weights):
+    if not isinstance(weights, dict) or not REQUIRED_WEIGHT_KEYS.issubset(weights):
+        return False
+    values = [weights[key] for key in REQUIRED_WEIGHT_KEYS]
+    return all(
+        isinstance(value, (int, float))
+        and not isinstance(value, bool)
+        and math.isfinite(value)
+        and value >= 0
+        for value in values
+    ) and sum(values) > 0
+
+
+def _has_oos_fold_weights(artifact):
+    folds = artifact.get("cv_results", {}).get("folds", [])
+    if not isinstance(folds, list) or not folds:
+        return False
+    for fold in folds:
+        validation_range = fold.get("date_ranges", {}).get("validation")
+        if not validation_range or not _weights_valid(fold.get("weights")):
+            return False
+    return True
+
+
+def validate_ml_artifact(artifact):
+    """Return machine-readable validity and provenance for an ML artifact."""
+    errors = []
+    if not isinstance(artifact, dict):
+        artifact = {}
+        errors.append("artifact_object")
+
+    schema_version = artifact.get("artifact_schema_version")
+    if schema_version != ML_ARTIFACT_SCHEMA_VERSION:
+        errors.append("artifact_schema_version")
+
+    score_version = artifact.get("score_version")
+    if score_version != SCORE_VERSION:
+        errors.append("score_version")
+
+    if not _weights_valid(artifact.get("weights")):
+        errors.append("weights")
+
+    provenance = artifact.get("provenance")
+    if not isinstance(provenance, dict):
+        provenance = {}
+        errors.append("provenance")
+    else:
+        required_provenance = {
+            "validation_method",
+            "label_horizon_days",
+            "weight_scope",
+            "training_date_range",
+            "trained_at",
+        }
+        if not required_provenance.issubset(provenance):
+            errors.append("provenance")
+        if provenance.get("validation_method") != "purged_expanding_window":
+            errors.append("purged_validation")
+        if provenance.get("label_horizon_days") != 365:
+            errors.append("label_horizon_days")
+        if provenance.get("weight_scope") != "full_history_fit":
+            errors.append("weight_scope")
+
+    return {
+        "valid": not errors,
+        "schema_version": schema_version,
+        "score_version": score_version,
+        "weight_scope": provenance.get("weight_scope"),
+        "has_oos_fold_weights": _has_oos_fold_weights(artifact),
+        "errors": list(dict.fromkeys(errors)),
+    }

ml/optimizer.py

@@ -26,7 +26,8 @@ from sklearn.metrics import (
 from sklearn.model_selection import TimeSeriesSplit
 from sklearn.preprocessing import StandardScaler
 
-from scoring.policy import SCORE_BRACKETS, score_in_bracket
+from scoring.policy import SCORE_BRACKETS, SCORE_VERSION, score_in_bracket
+from ml.artifacts import ML_ARTIFACT_SCHEMA_VERSION
 
 logging.basicConfig(
     level=logging.INFO,
@@ -478,8 +479,12 @@ def train_model(rows):
     comparison = run_comparison(rows, weights)
     out_of_sample_comparison = run_out_of_sample_comparison(labeled, fold_results)
 
-    # Build output
+    # Build output. Final weights are fitted on all labeled history for live use;
+    # only the fold weights below are valid for OOS comparisons.
+    trained_at = datetime.now(tz=__import__('datetime').timezone.utc).isoformat()
     result = {
+        "artifact_schema_version": ML_ARTIFACT_SCHEMA_VERSION,
+        "score_version": SCORE_VERSION,
         "weights": weights,
         "feature_importances": {name: round(float(imp), 6) for name, imp in feat_imp},
         "cv_results": {
@@ -501,9 +506,19 @@ def train_model(rows):
             "date_range": f"{labeled[0]['date']} to {labeled[-1]['date']}",
             "model": "GradientBoostingClassifier",
         },
+        "provenance": {
+            "validation_method": "purged_expanding_window",
+            "label_horizon_days": LABEL_HORIZON_DAYS,
+            "weight_scope": "full_history_fit",
+            "training_date_range": {
+                "start": labeled[0]["date"],
+                "end": labeled[-1]["date"],
+            },
+            "trained_at": trained_at,
+        },
         "comparison": comparison,
         "out_of_sample_comparison": out_of_sample_comparison,
-        "trained_at": datetime.now(tz=__import__('datetime').timezone.utc).isoformat(),
+        "trained_at": trained_at,
     }
 
     return result

scoring/engine.py

@@ -5,6 +5,7 @@ import os
 import logging
 
 from scoring.policy import SCORE_VERSION, assessment_for_score
+from ml.artifacts import validate_ml_artifact
 
 log = logging.getLogger(__name__)
 
@@ -580,16 +581,30 @@ _ML_KEY_MAP = {
 }
 
 
+_ml_artifact_status = {"valid": False, "errors": ["not_loaded"]}
+
+
 def load_ml_weights():
-    """Load ML-optimized weights from config."""
+    """Load weights only when their schema and training provenance are valid."""
+    global _ml_artifact_status
     try:
         with open(ML_WEIGHTS_PATH) as f:
             data = json.load(f)
+        _ml_artifact_status = validate_ml_artifact(data)
+        if not _ml_artifact_status["valid"]:
+            log.error("Rejected invalid ML artifact: %s", ", ".join(_ml_artifact_status["errors"]))
+            return {}
         return data.get("weights", {})
-    except Exception:
+    except Exception as exc:
+        _ml_artifact_status = {"valid": False, "errors": [f"load_error:{exc}"]}
         return {}
 
 
+def get_ml_artifact_status():
+    """Return the status from the most recent artifact load attempt."""
+    return dict(_ml_artifact_status)
+
+
 def score_all_ml(metrics):
     """Score all metrics using ML-optimized weights.
 
@@ -604,7 +619,12 @@ def score_all_ml(metrics):
     if not ml_weights:
         # Fallback to classic if no ML weights available
         classic["ml_mode"] = False
-        classic["ml_error"] = "ML weights not found — run ml/optimizer.py"
+        status = get_ml_artifact_status()
+        if status.get("errors") and status["errors"] != ["not_loaded"]:
+            classic["ml_error"] = "ML artifact invalid: " + ", ".join(status["errors"])
+        else:
+            classic["ml_error"] = "ML weights not found — run ml/optimizer.py"
+        classic["ml_artifact"] = status
         return classic
 
     results = classic["metrics"]

tests/test_ml_artifacts.py

@@ -0,0 +1,58 @@
+import json
+from pathlib import Path
+
+from ml import artifacts
+from scoring import engine
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+
+def _valid_artifact():
+    return {
+        "artifact_schema_version": artifacts.ML_ARTIFACT_SCHEMA_VERSION,
+        "score_version": artifacts.SCORE_VERSION,
+        "weights": {key: 1 / len(artifacts.REQUIRED_WEIGHT_KEYS) for key in artifacts.REQUIRED_WEIGHT_KEYS},
+        "provenance": {
+            "validation_method": "purged_expanding_window",
+            "label_horizon_days": 365,
+            "weight_scope": "full_history_fit",
+            "training_date_range": {"start": "2018-02-01", "end": "2025-03-21"},
+            "trained_at": "2026-07-01T00:00:00+00:00",
+        },
+    }
+
+
+def test_pre_purge_repository_artifact_is_rejected_with_actionable_status():
+    artifact_path = REPO_ROOT / "config" / "ml_weights.json"
+    artifact = json.loads(artifact_path.read_text())
+
+    status = artifacts.validate_ml_artifact(artifact)
+
+    assert status["valid"] is False
+    assert status["schema_version"] is None
+    assert "artifact_schema_version" in status["errors"]
+
+
+def test_valid_artifact_requires_schema_score_version_and_purged_provenance():
+    artifact = _valid_artifact()
+
+    status = artifacts.validate_ml_artifact(artifact)
+
+    assert status == {
+        "valid": True,
+        "schema_version": artifacts.ML_ARTIFACT_SCHEMA_VERSION,
+        "score_version": artifacts.SCORE_VERSION,
+        "weight_scope": "full_history_fit",
+        "has_oos_fold_weights": False,
+        "errors": [],
+    }
+
+
+def test_live_scoring_refuses_schema_less_weights(tmp_path, monkeypatch):
+    path = tmp_path / "ml_weights.json"
+    path.write_text(json.dumps({"weights": {"fear_greed": 1.0}}))
+    monkeypatch.setattr(engine, "ML_WEIGHTS_PATH", str(path))
+
+    assert engine.load_ml_weights() == {}
+    assert engine.get_ml_artifact_status()["valid"] is False

tests/test_scoring_engine_ml.py

@@ -59,5 +59,6 @@ def test_score_all_ml_preserves_classic_fallback_when_weights_missing(monkeypatc
     scored = engine.score_all_ml(_complete_metrics())
 
     assert scored["ml_mode"] is False
-    assert scored["ml_error"] == "ML weights not found — run ml/optimizer.py"
+    assert scored["ml_error"]
+    assert scored["ml_artifact"]["valid"] is False
     assert "classic_score" not in scored