diff --git a/backtesting/engine.py b/backtesting/engine.py index 859a2a9..a584d82 100644 --- a/backtesting/engine.py +++ b/backtesting/engine.py @@ -117,8 +117,8 @@ def _compute_ath_series(price_lookup, dates): return drawdowns -def _load_ml_weights(): - """Load only schema/provenance-valid ML weights.""" +def _load_ml_artifact(): + """Load an ML artifact and return it with validation status.""" ml_path = _os.path.join(_os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))), "config", "ml_weights.json") try: with open(ml_path) as f: @@ -126,10 +126,61 @@ def _load_ml_weights(): status = validate_ml_artifact(data) if not status["valid"]: log.error("Rejected invalid ML artifact: %s", ", ".join(status["errors"])) - return {} - return data.get("weights", {}) - except Exception: - return {} + return None, status + return data, status + except Exception as exc: + return None, {"valid": False, "errors": [f"load_error:{exc}"]} + + +def _build_ml_backtest_plan(artifact): + """Choose OOS fold weights when available; otherwise mark final weights in-sample.""" + status = validate_ml_artifact(artifact) + if not status["valid"]: + raise ValueError("invalid ML artifact: " + ", ".join(status["errors"])) + + if status["has_oos_fold_weights"]: + folds = [] + for fold in artifact["cv_results"]["folds"]: + start, separator, end = fold["date_ranges"]["validation"].partition(" to ") + if not separator: + raise ValueError("invalid validation date range") + folds.append({ + "fold": fold.get("fold"), + "start": start, + "end": end, + "weights": fold["weights"], + }) + return { + "evaluation_scope": "out_of_sample_validation_folds", + "is_out_of_sample": True, + "weighting_source": "fold_specific_weights", + "folds": folds, + "weights": None, + } + + return { + "evaluation_scope": "in_sample_full_history_weights", + "is_out_of_sample": False, + "weighting_source": "final_full_history_weights", + "folds": [], + "weights": artifact["weights"], + } + + +def _weights_for_backtest_date(date, plan): + """Return date-appropriate weights and fold number for an ML plan.""" + if plan["is_out_of_sample"]: + for fold in plan["folds"]: + if fold["start"] <= date <= fold["end"]: + return fold["weights"], fold["fold"] + return None, None + return plan["weights"], None + + +def _load_ml_weights(): + """Compatibility helper returning valid final weights only.""" + artifact, _ = _load_ml_artifact() + return artifact.get("weights", {}) if artifact else {} # ML weight key mapping (backtest metric keys -> ML weight keys) _BT_ML_KEY_MAP = { @@ -285,16 +336,27 @@ def run_backtest(ml_mode=False): log.info("Computing forward returns...") fwd_returns = compute_forward_returns(price_lookup, all_dates) - # Load ML weights if in ML mode - ml_weights = _load_ml_weights() if ml_mode else None - if ml_mode and not ml_weights: - log.warning("ML mode requested but no weights found — falling back to equal weights") - ml_weights = None + # Build an explicit evaluation plan. Fold-specific validation weights are OOS; + # final weights fitted on full history are never represented as OOS. + ml_plan = None + ml_artifact_status = None + if ml_mode: + ml_artifact, ml_artifact_status = _load_ml_artifact() + if ml_artifact: + ml_plan = _build_ml_backtest_plan(ml_artifact) + else: + log.warning("ML mode requested with invalid artifact — falling back to equal weights") # Score each day log.info("Scoring %d days...", len(all_dates)) daily_scores = [] for d in all_dates: + ml_weights = None + ml_fold = None + if ml_plan: + ml_weights, ml_fold = _weights_for_backtest_date(d, ml_plan) + if ml_plan["is_out_of_sample"] and ml_weights is None: + continue composite, details, n_metrics = score_day(d, index, drawdowns, ml_weights=ml_weights) if composite is not None and n_metrics >= 3: # Require at least 3 metrics price = price_lookup.get(d) @@ -312,6 +374,8 @@ def run_backtest(ml_mode=False): "forward_returns": fwd_returns.get(d, {}), "metric_values": metric_values, } + if ml_fold is not None: + entry["ml_fold"] = ml_fold daily_scores.append(entry) if not daily_scores: @@ -498,6 +562,27 @@ def run_backtest(ml_mode=False): entry["metrics"] = metric_vals chart_data.append(entry) + if not ml_mode: + ml_evaluation = {"requested": False, "is_out_of_sample": False} + elif ml_plan: + ml_evaluation = { + "requested": True, + "evaluation_scope": ml_plan["evaluation_scope"], + "is_out_of_sample": ml_plan["is_out_of_sample"], + "weighting_source": ml_plan["weighting_source"], + "folds": len(ml_plan["folds"]), + "artifact": ml_artifact_status, + } + else: + ml_evaluation = { + "requested": True, + "evaluation_scope": "equal_weight_fallback", + "is_out_of_sample": False, + "weighting_source": "none_invalid_artifact", + "folds": 0, + "artifact": ml_artifact_status, + } + result = { "date_range": {"start": daily_scores[0]["date"], "end": daily_scores[-1]["date"]}, "total_days_scored": len(daily_scores), @@ -506,6 +591,7 @@ def run_backtest(ml_mode=False): "current_context": current_context, "chart_data": chart_data, "ml_mode": ml_mode, + "ml_evaluation": ml_evaluation, "score_version": SCORE_VERSION, "computed_at": datetime.utcnow().isoformat() + "Z", } diff --git a/tests/test_backtest_ml_scope.py b/tests/test_backtest_ml_scope.py new file mode 100644 index 0000000..83b7411 --- /dev/null +++ b/tests/test_backtest_ml_scope.py @@ -0,0 +1,64 @@ +from backtesting import engine +from ml.artifacts import ML_ARTIFACT_SCHEMA_VERSION, REQUIRED_WEIGHT_KEYS +from scoring.policy import SCORE_VERSION + + +def _weights(focus): + weights = {key: 0.0 for key in REQUIRED_WEIGHT_KEYS} + weights[focus] = 1.0 + return weights + + +def _artifact(with_folds=True): + artifact = { + "artifact_schema_version": ML_ARTIFACT_SCHEMA_VERSION, + "score_version": SCORE_VERSION, + "weights": _weights("fear_greed"), + "provenance": { + "validation_method": "purged_expanding_window", + "label_horizon_days": 365, + "weight_scope": "full_history_fit", + "training_date_range": {"start": "2018-01-01", "end": "2024-01-01"}, + "trained_at": "2026-07-01T00:00:00+00:00", + }, + "cv_results": {"folds": []}, + } + if with_folds: + artifact["cv_results"]["folds"] = [ + { + "fold": 1, + "weights": _weights("drawdown"), + "date_ranges": {"validation": "2020-01-01 to 2020-12-31"}, + }, + { + "fold": 2, + "weights": _weights("nupl"), + "date_ranges": {"validation": "2021-01-01 to 2021-12-31"}, + }, + ] + return artifact + + +def test_ml_backtest_plan_prefers_fold_weights_and_marks_them_oos(): + plan = engine._build_ml_backtest_plan(_artifact(with_folds=True)) + + weights, fold = engine._weights_for_backtest_date("2021-06-01", plan) + + assert weights == _weights("nupl") + assert fold == 2 + assert plan["evaluation_scope"] == "out_of_sample_validation_folds" + assert plan["is_out_of_sample"] is True + assert plan["weighting_source"] == "fold_specific_weights" + assert engine._weights_for_backtest_date("2019-12-31", plan) == (None, None) + + +def test_full_history_weights_are_explicitly_not_oos(): + plan = engine._build_ml_backtest_plan(_artifact(with_folds=False)) + + weights, fold = engine._weights_for_backtest_date("2021-06-01", plan) + + assert weights == _weights("fear_greed") + assert fold is None + assert plan["evaluation_scope"] == "in_sample_full_history_weights" + assert plan["is_out_of_sample"] is False + assert plan["weighting_source"] == "final_full_history_weights"