fix: distinguish OOS ML backtest weights
This commit is contained in:
Hermes Agent
+97
-11
@@ -117,8 +117,8 @@ def _compute_ath_series(price_lookup, dates):
|
||||
return drawdowns
|
||||
|
||||
|
||||
def _load_ml_weights():
|
||||
"""Load only schema/provenance-valid ML weights."""
|
||||
def _load_ml_artifact():
|
||||
"""Load an ML artifact and return it with validation status."""
|
||||
ml_path = _os.path.join(_os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))), "config", "ml_weights.json")
|
||||
try:
|
||||
with open(ml_path) as f:
|
||||
@@ -126,10 +126,61 @@ def _load_ml_weights():
|
||||
status = validate_ml_artifact(data)
|
||||
if not status["valid"]:
|
||||
log.error("Rejected invalid ML artifact: %s", ", ".join(status["errors"]))
|
||||
return {}
|
||||
return data.get("weights", {})
|
||||
except Exception:
|
||||
return {}
|
||||
return None, status
|
||||
return data, status
|
||||
except Exception as exc:
|
||||
return None, {"valid": False, "errors": [f"load_error:{exc}"]}
|
||||
|
||||
|
||||
def _build_ml_backtest_plan(artifact):
|
||||
"""Choose OOS fold weights when available; otherwise mark final weights in-sample."""
|
||||
status = validate_ml_artifact(artifact)
|
||||
if not status["valid"]:
|
||||
raise ValueError("invalid ML artifact: " + ", ".join(status["errors"]))
|
||||
|
||||
if status["has_oos_fold_weights"]:
|
||||
folds = []
|
||||
for fold in artifact["cv_results"]["folds"]:
|
||||
start, separator, end = fold["date_ranges"]["validation"].partition(" to ")
|
||||
if not separator:
|
||||
raise ValueError("invalid validation date range")
|
||||
folds.append({
|
||||
"fold": fold.get("fold"),
|
||||
"start": start,
|
||||
"end": end,
|
||||
"weights": fold["weights"],
|
||||
})
|
||||
return {
|
||||
"evaluation_scope": "out_of_sample_validation_folds",
|
||||
"is_out_of_sample": True,
|
||||
"weighting_source": "fold_specific_weights",
|
||||
"folds": folds,
|
||||
"weights": None,
|
||||
}
|
||||
|
||||
return {
|
||||
"evaluation_scope": "in_sample_full_history_weights",
|
||||
"is_out_of_sample": False,
|
||||
"weighting_source": "final_full_history_weights",
|
||||
"folds": [],
|
||||
"weights": artifact["weights"],
|
||||
}
|
||||
|
||||
|
||||
def _weights_for_backtest_date(date, plan):
|
||||
"""Return date-appropriate weights and fold number for an ML plan."""
|
||||
if plan["is_out_of_sample"]:
|
||||
for fold in plan["folds"]:
|
||||
if fold["start"] <= date <= fold["end"]:
|
||||
return fold["weights"], fold["fold"]
|
||||
return None, None
|
||||
return plan["weights"], None
|
||||
|
||||
|
||||
def _load_ml_weights():
|
||||
"""Compatibility helper returning valid final weights only."""
|
||||
artifact, _ = _load_ml_artifact()
|
||||
return artifact.get("weights", {}) if artifact else {}
|
||||
|
||||
# ML weight key mapping (backtest metric keys -> ML weight keys)
|
||||
_BT_ML_KEY_MAP = {
|
||||
@@ -285,16 +336,27 @@ def run_backtest(ml_mode=False):
|
||||
log.info("Computing forward returns...")
|
||||
fwd_returns = compute_forward_returns(price_lookup, all_dates)
|
||||
|
||||
# Load ML weights if in ML mode
|
||||
ml_weights = _load_ml_weights() if ml_mode else None
|
||||
if ml_mode and not ml_weights:
|
||||
log.warning("ML mode requested but no weights found — falling back to equal weights")
|
||||
ml_weights = None
|
||||
# Build an explicit evaluation plan. Fold-specific validation weights are OOS;
|
||||
# final weights fitted on full history are never represented as OOS.
|
||||
ml_plan = None
|
||||
ml_artifact_status = None
|
||||
if ml_mode:
|
||||
ml_artifact, ml_artifact_status = _load_ml_artifact()
|
||||
if ml_artifact:
|
||||
ml_plan = _build_ml_backtest_plan(ml_artifact)
|
||||
else:
|
||||
log.warning("ML mode requested with invalid artifact — falling back to equal weights")
|
||||
|
||||
# Score each day
|
||||
log.info("Scoring %d days...", len(all_dates))
|
||||
daily_scores = []
|
||||
for d in all_dates:
|
||||
ml_weights = None
|
||||
ml_fold = None
|
||||
if ml_plan:
|
||||
ml_weights, ml_fold = _weights_for_backtest_date(d, ml_plan)
|
||||
if ml_plan["is_out_of_sample"] and ml_weights is None:
|
||||
continue
|
||||
composite, details, n_metrics = score_day(d, index, drawdowns, ml_weights=ml_weights)
|
||||
if composite is not None and n_metrics >= 3: # Require at least 3 metrics
|
||||
price = price_lookup.get(d)
|
||||
@@ -312,6 +374,8 @@ def run_backtest(ml_mode=False):
|
||||
"forward_returns": fwd_returns.get(d, {}),
|
||||
"metric_values": metric_values,
|
||||
}
|
||||
if ml_fold is not None:
|
||||
entry["ml_fold"] = ml_fold
|
||||
daily_scores.append(entry)
|
||||
|
||||
if not daily_scores:
|
||||
@@ -498,6 +562,27 @@ def run_backtest(ml_mode=False):
|
||||
entry["metrics"] = metric_vals
|
||||
chart_data.append(entry)
|
||||
|
||||
if not ml_mode:
|
||||
ml_evaluation = {"requested": False, "is_out_of_sample": False}
|
||||
elif ml_plan:
|
||||
ml_evaluation = {
|
||||
"requested": True,
|
||||
"evaluation_scope": ml_plan["evaluation_scope"],
|
||||
"is_out_of_sample": ml_plan["is_out_of_sample"],
|
||||
"weighting_source": ml_plan["weighting_source"],
|
||||
"folds": len(ml_plan["folds"]),
|
||||
"artifact": ml_artifact_status,
|
||||
}
|
||||
else:
|
||||
ml_evaluation = {
|
||||
"requested": True,
|
||||
"evaluation_scope": "equal_weight_fallback",
|
||||
"is_out_of_sample": False,
|
||||
"weighting_source": "none_invalid_artifact",
|
||||
"folds": 0,
|
||||
"artifact": ml_artifact_status,
|
||||
}
|
||||
|
||||
result = {
|
||||
"date_range": {"start": daily_scores[0]["date"], "end": daily_scores[-1]["date"]},
|
||||
"total_days_scored": len(daily_scores),
|
||||
@@ -506,6 +591,7 @@ def run_backtest(ml_mode=False):
|
||||
"current_context": current_context,
|
||||
"chart_data": chart_data,
|
||||
"ml_mode": ml_mode,
|
||||
"ml_evaluation": ml_evaluation,
|
||||
"score_version": SCORE_VERSION,
|
||||
"computed_at": datetime.utcnow().isoformat() + "Z",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
from backtesting import engine
|
||||
from ml.artifacts import ML_ARTIFACT_SCHEMA_VERSION, REQUIRED_WEIGHT_KEYS
|
||||
from scoring.policy import SCORE_VERSION
|
||||
|
||||
|
||||
def _weights(focus):
|
||||
weights = {key: 0.0 for key in REQUIRED_WEIGHT_KEYS}
|
||||
weights[focus] = 1.0
|
||||
return weights
|
||||
|
||||
|
||||
def _artifact(with_folds=True):
|
||||
artifact = {
|
||||
"artifact_schema_version": ML_ARTIFACT_SCHEMA_VERSION,
|
||||
"score_version": SCORE_VERSION,
|
||||
"weights": _weights("fear_greed"),
|
||||
"provenance": {
|
||||
"validation_method": "purged_expanding_window",
|
||||
"label_horizon_days": 365,
|
||||
"weight_scope": "full_history_fit",
|
||||
"training_date_range": {"start": "2018-01-01", "end": "2024-01-01"},
|
||||
"trained_at": "2026-07-01T00:00:00+00:00",
|
||||
},
|
||||
"cv_results": {"folds": []},
|
||||
}
|
||||
if with_folds:
|
||||
artifact["cv_results"]["folds"] = [
|
||||
{
|
||||
"fold": 1,
|
||||
"weights": _weights("drawdown"),
|
||||
"date_ranges": {"validation": "2020-01-01 to 2020-12-31"},
|
||||
},
|
||||
{
|
||||
"fold": 2,
|
||||
"weights": _weights("nupl"),
|
||||
"date_ranges": {"validation": "2021-01-01 to 2021-12-31"},
|
||||
},
|
||||
]
|
||||
return artifact
|
||||
|
||||
|
||||
def test_ml_backtest_plan_prefers_fold_weights_and_marks_them_oos():
|
||||
plan = engine._build_ml_backtest_plan(_artifact(with_folds=True))
|
||||
|
||||
weights, fold = engine._weights_for_backtest_date("2021-06-01", plan)
|
||||
|
||||
assert weights == _weights("nupl")
|
||||
assert fold == 2
|
||||
assert plan["evaluation_scope"] == "out_of_sample_validation_folds"
|
||||
assert plan["is_out_of_sample"] is True
|
||||
assert plan["weighting_source"] == "fold_specific_weights"
|
||||
assert engine._weights_for_backtest_date("2019-12-31", plan) == (None, None)
|
||||
|
||||
|
||||
def test_full_history_weights_are_explicitly_not_oos():
|
||||
plan = engine._build_ml_backtest_plan(_artifact(with_folds=False))
|
||||
|
||||
weights, fold = engine._weights_for_backtest_date("2021-06-01", plan)
|
||||
|
||||
assert weights == _weights("fear_greed")
|
||||
assert fold is None
|
||||
assert plan["evaluation_scope"] == "in_sample_full_history_weights"
|
||||
assert plan["is_out_of_sample"] is False
|
||||
assert plan["weighting_source"] == "final_full_history_weights"
|
||||
Reference in New Issue
Block a user