fix: distinguish OOS ML backtest weights

backtesting/engine.py

@@ -117,8 +117,8 @@ def _compute_ath_series(price_lookup, dates):
     return drawdowns
 
 
-def _load_ml_weights():
-    """Load only schema/provenance-valid ML weights."""
+def _load_ml_artifact():
+    """Load an ML artifact and return it with validation status."""
     ml_path = _os.path.join(_os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))), "config", "ml_weights.json")
     try:
         with open(ml_path) as f:
@@ -126,10 +126,61 @@ def _load_ml_weights():
         status = validate_ml_artifact(data)
         if not status["valid"]:
             log.error("Rejected invalid ML artifact: %s", ", ".join(status["errors"]))
-            return {}
-        return data.get("weights", {})
-    except Exception:
-        return {}
+            return None, status
+        return data, status
+    except Exception as exc:
+        return None, {"valid": False, "errors": [f"load_error:{exc}"]}
+
+
+def _build_ml_backtest_plan(artifact):
+    """Choose OOS fold weights when available; otherwise mark final weights in-sample."""
+    status = validate_ml_artifact(artifact)
+    if not status["valid"]:
+        raise ValueError("invalid ML artifact: " + ", ".join(status["errors"]))
+
+    if status["has_oos_fold_weights"]:
+        folds = []
+        for fold in artifact["cv_results"]["folds"]:
+            start, separator, end = fold["date_ranges"]["validation"].partition(" to ")
+            if not separator:
+                raise ValueError("invalid validation date range")
+            folds.append({
+                "fold": fold.get("fold"),
+                "start": start,
+                "end": end,
+                "weights": fold["weights"],
+            })
+        return {
+            "evaluation_scope": "out_of_sample_validation_folds",
+            "is_out_of_sample": True,
+            "weighting_source": "fold_specific_weights",
+            "folds": folds,
+            "weights": None,
+        }
+
+    return {
+        "evaluation_scope": "in_sample_full_history_weights",
+        "is_out_of_sample": False,
+        "weighting_source": "final_full_history_weights",
+        "folds": [],
+        "weights": artifact["weights"],
+    }
+
+
+def _weights_for_backtest_date(date, plan):
+    """Return date-appropriate weights and fold number for an ML plan."""
+    if plan["is_out_of_sample"]:
+        for fold in plan["folds"]:
+            if fold["start"] <= date <= fold["end"]:
+                return fold["weights"], fold["fold"]
+        return None, None
+    return plan["weights"], None
+
+
+def _load_ml_weights():
+    """Compatibility helper returning valid final weights only."""
+    artifact, _ = _load_ml_artifact()
+    return artifact.get("weights", {}) if artifact else {}
 
 # ML weight key mapping (backtest metric keys -> ML weight keys)
 _BT_ML_KEY_MAP = {
@@ -285,16 +336,27 @@ def run_backtest(ml_mode=False):
     log.info("Computing forward returns...")
     fwd_returns = compute_forward_returns(price_lookup, all_dates)
 
-    # Load ML weights if in ML mode
-    ml_weights = _load_ml_weights() if ml_mode else None
-    if ml_mode and not ml_weights:
-        log.warning("ML mode requested but no weights found — falling back to equal weights")
-        ml_weights = None
+    # Build an explicit evaluation plan. Fold-specific validation weights are OOS;
+    # final weights fitted on full history are never represented as OOS.
+    ml_plan = None
+    ml_artifact_status = None
+    if ml_mode:
+        ml_artifact, ml_artifact_status = _load_ml_artifact()
+        if ml_artifact:
+            ml_plan = _build_ml_backtest_plan(ml_artifact)
+        else:
+            log.warning("ML mode requested with invalid artifact — falling back to equal weights")
 
     # Score each day
     log.info("Scoring %d days...", len(all_dates))
     daily_scores = []
     for d in all_dates:
+        ml_weights = None
+        ml_fold = None
+        if ml_plan:
+            ml_weights, ml_fold = _weights_for_backtest_date(d, ml_plan)
+            if ml_plan["is_out_of_sample"] and ml_weights is None:
+                continue
         composite, details, n_metrics = score_day(d, index, drawdowns, ml_weights=ml_weights)
         if composite is not None and n_metrics >= 3:  # Require at least 3 metrics
             price = price_lookup.get(d)
@@ -312,6 +374,8 @@ def run_backtest(ml_mode=False):
                 "forward_returns": fwd_returns.get(d, {}),
                 "metric_values": metric_values,
             }
+            if ml_fold is not None:
+                entry["ml_fold"] = ml_fold
             daily_scores.append(entry)
 
     if not daily_scores:
@@ -498,6 +562,27 @@ def run_backtest(ml_mode=False):
                 entry["metrics"] = metric_vals
             chart_data.append(entry)
 
+    if not ml_mode:
+        ml_evaluation = {"requested": False, "is_out_of_sample": False}
+    elif ml_plan:
+        ml_evaluation = {
+            "requested": True,
+            "evaluation_scope": ml_plan["evaluation_scope"],
+            "is_out_of_sample": ml_plan["is_out_of_sample"],
+            "weighting_source": ml_plan["weighting_source"],
+            "folds": len(ml_plan["folds"]),
+            "artifact": ml_artifact_status,
+        }
+    else:
+        ml_evaluation = {
+            "requested": True,
+            "evaluation_scope": "equal_weight_fallback",
+            "is_out_of_sample": False,
+            "weighting_source": "none_invalid_artifact",
+            "folds": 0,
+            "artifact": ml_artifact_status,
+        }
+
     result = {
         "date_range": {"start": daily_scores[0]["date"], "end": daily_scores[-1]["date"]},
         "total_days_scored": len(daily_scores),
@@ -506,6 +591,7 @@ def run_backtest(ml_mode=False):
         "current_context": current_context,
         "chart_data": chart_data,
         "ml_mode": ml_mode,
+        "ml_evaluation": ml_evaluation,
         "score_version": SCORE_VERSION,
         "computed_at": datetime.utcnow().isoformat() + "Z",
     }