fix: distinguish OOS ML backtest weights

backtesting/engine.py

@@ -117,8 +117,8 @@ def _compute_ath_series(price_lookup, dates):
     return drawdowns
 
 
-def _load_ml_weights():
+def _load_ml_artifact():
-    """Load only schema/provenance-valid ML weights."""
+    """Load an ML artifact and return it with validation status."""
     ml_path = _os.path.join(_os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))), "config", "ml_weights.json")
     try:
         with open(ml_path) as f:
@@ -126,10 +126,61 @@ def _load_ml_weights():
         status = validate_ml_artifact(data)
         if not status["valid"]:
             log.error("Rejected invalid ML artifact: %s", ", ".join(status["errors"]))
-            return {}
+            return None, status
-        return data.get("weights", {})
+        return data, status
-    except Exception:
+    except Exception as exc:
-        return {}
+        return None, {"valid": False, "errors": [f"load_error:{exc}"]}
+
+
+def _build_ml_backtest_plan(artifact):
+    """Choose OOS fold weights when available; otherwise mark final weights in-sample."""
+    status = validate_ml_artifact(artifact)
+    if not status["valid"]:
+        raise ValueError("invalid ML artifact: " + ", ".join(status["errors"]))
+
+    if status["has_oos_fold_weights"]:
+        folds = []
+        for fold in artifact["cv_results"]["folds"]:
+            start, separator, end = fold["date_ranges"]["validation"].partition(" to ")
+            if not separator:
+                raise ValueError("invalid validation date range")
+            folds.append({
+                "fold": fold.get("fold"),
+                "start": start,
+                "end": end,
+                "weights": fold["weights"],
+            })
+        return {
+            "evaluation_scope": "out_of_sample_validation_folds",
+            "is_out_of_sample": True,
+            "weighting_source": "fold_specific_weights",
+            "folds": folds,
+            "weights": None,
+        }
+
+    return {
+        "evaluation_scope": "in_sample_full_history_weights",
+        "is_out_of_sample": False,
+        "weighting_source": "final_full_history_weights",
+        "folds": [],
+        "weights": artifact["weights"],
+    }
+
+
+def _weights_for_backtest_date(date, plan):
+    """Return date-appropriate weights and fold number for an ML plan."""
+    if plan["is_out_of_sample"]:
+        for fold in plan["folds"]:
+            if fold["start"] <= date <= fold["end"]:
+                return fold["weights"], fold["fold"]
+        return None, None
+    return plan["weights"], None
+
+
+def _load_ml_weights():
+    """Compatibility helper returning valid final weights only."""
+    artifact, _ = _load_ml_artifact()
+    return artifact.get("weights", {}) if artifact else {}
 
 # ML weight key mapping (backtest metric keys -> ML weight keys)
 _BT_ML_KEY_MAP = {
@@ -285,16 +336,27 @@ def run_backtest(ml_mode=False):
     log.info("Computing forward returns...")
     fwd_returns = compute_forward_returns(price_lookup, all_dates)
 
-    # Load ML weights if in ML mode
+    # Build an explicit evaluation plan. Fold-specific validation weights are OOS;
-    ml_weights = _load_ml_weights() if ml_mode else None
+    # final weights fitted on full history are never represented as OOS.
-    if ml_mode and not ml_weights:
+    ml_plan = None
-        log.warning("ML mode requested but no weights found — falling back to equal weights")
+    ml_artifact_status = None
-        ml_weights = None
+    if ml_mode:
+        ml_artifact, ml_artifact_status = _load_ml_artifact()
+        if ml_artifact:
+            ml_plan = _build_ml_backtest_plan(ml_artifact)
+        else:
+            log.warning("ML mode requested with invalid artifact — falling back to equal weights")
 
     # Score each day
     log.info("Scoring %d days...", len(all_dates))
     daily_scores = []
     for d in all_dates:
+        ml_weights = None
+        ml_fold = None
+        if ml_plan:
+            ml_weights, ml_fold = _weights_for_backtest_date(d, ml_plan)
+            if ml_plan["is_out_of_sample"] and ml_weights is None:
+                continue
         composite, details, n_metrics = score_day(d, index, drawdowns, ml_weights=ml_weights)
         if composite is not None and n_metrics >= 3:  # Require at least 3 metrics
             price = price_lookup.get(d)
@@ -312,6 +374,8 @@ def run_backtest(ml_mode=False):
                 "forward_returns": fwd_returns.get(d, {}),
                 "metric_values": metric_values,
             }
+            if ml_fold is not None:
+                entry["ml_fold"] = ml_fold
             daily_scores.append(entry)
 
     if not daily_scores:
@@ -498,6 +562,27 @@ def run_backtest(ml_mode=False):
                 entry["metrics"] = metric_vals
             chart_data.append(entry)
 
+    if not ml_mode:
+        ml_evaluation = {"requested": False, "is_out_of_sample": False}
+    elif ml_plan:
+        ml_evaluation = {
+            "requested": True,
+            "evaluation_scope": ml_plan["evaluation_scope"],
+            "is_out_of_sample": ml_plan["is_out_of_sample"],
+            "weighting_source": ml_plan["weighting_source"],
+            "folds": len(ml_plan["folds"]),
+            "artifact": ml_artifact_status,
+        }
+    else:
+        ml_evaluation = {
+            "requested": True,
+            "evaluation_scope": "equal_weight_fallback",
+            "is_out_of_sample": False,
+            "weighting_source": "none_invalid_artifact",
+            "folds": 0,
+            "artifact": ml_artifact_status,
+        }
+
     result = {
         "date_range": {"start": daily_scores[0]["date"], "end": daily_scores[-1]["date"]},
         "total_days_scored": len(daily_scores),
@@ -506,6 +591,7 @@ def run_backtest(ml_mode=False):
         "current_context": current_context,
         "chart_data": chart_data,
         "ml_mode": ml_mode,
+        "ml_evaluation": ml_evaluation,
         "score_version": SCORE_VERSION,
         "computed_at": datetime.utcnow().isoformat() + "Z",
     }

tests/test_backtest_ml_scope.py

@@ -0,0 +1,64 @@
+from backtesting import engine
+from ml.artifacts import ML_ARTIFACT_SCHEMA_VERSION, REQUIRED_WEIGHT_KEYS
+from scoring.policy import SCORE_VERSION
+
+
+def _weights(focus):
+    weights = {key: 0.0 for key in REQUIRED_WEIGHT_KEYS}
+    weights[focus] = 1.0
+    return weights
+
+
+def _artifact(with_folds=True):
+    artifact = {
+        "artifact_schema_version": ML_ARTIFACT_SCHEMA_VERSION,
+        "score_version": SCORE_VERSION,
+        "weights": _weights("fear_greed"),
+        "provenance": {
+            "validation_method": "purged_expanding_window",
+            "label_horizon_days": 365,
+            "weight_scope": "full_history_fit",
+            "training_date_range": {"start": "2018-01-01", "end": "2024-01-01"},
+            "trained_at": "2026-07-01T00:00:00+00:00",
+        },
+        "cv_results": {"folds": []},
+    }
+    if with_folds:
+        artifact["cv_results"]["folds"] = [
+            {
+                "fold": 1,
+                "weights": _weights("drawdown"),
+                "date_ranges": {"validation": "2020-01-01 to 2020-12-31"},
+            },
+            {
+                "fold": 2,
+                "weights": _weights("nupl"),
+                "date_ranges": {"validation": "2021-01-01 to 2021-12-31"},
+            },
+        ]
+    return artifact
+
+
+def test_ml_backtest_plan_prefers_fold_weights_and_marks_them_oos():
+    plan = engine._build_ml_backtest_plan(_artifact(with_folds=True))
+
+    weights, fold = engine._weights_for_backtest_date("2021-06-01", plan)
+
+    assert weights == _weights("nupl")
+    assert fold == 2
+    assert plan["evaluation_scope"] == "out_of_sample_validation_folds"
+    assert plan["is_out_of_sample"] is True
+    assert plan["weighting_source"] == "fold_specific_weights"
+    assert engine._weights_for_backtest_date("2019-12-31", plan) == (None, None)
+
+
+def test_full_history_weights_are_explicitly_not_oos():
+    plan = engine._build_ml_backtest_plan(_artifact(with_folds=False))
+
+    weights, fold = engine._weights_for_backtest_date("2021-06-01", plan)
+
+    assert weights == _weights("fear_greed")
+    assert fold is None
+    assert plan["evaluation_scope"] == "in_sample_full_history_weights"
+    assert plan["is_out_of_sample"] is False
+    assert plan["weighting_source"] == "final_full_history_weights"