fix: distinguish OOS ML backtest weights

2026-07-26 23:07:24 +00:00
parent eb8c01611c
commit 510b2587ca
2 changed files with 161 additions and 11 deletions
@@ -0,0 +1,64 @@
+from backtesting import engine
+from ml.artifacts import ML_ARTIFACT_SCHEMA_VERSION, REQUIRED_WEIGHT_KEYS
+from scoring.policy import SCORE_VERSION
+
+
+def _weights(focus):
+    weights = {key: 0.0 for key in REQUIRED_WEIGHT_KEYS}
+    weights[focus] = 1.0
+    return weights
+
+
+def _artifact(with_folds=True):
+    artifact = {
+        "artifact_schema_version": ML_ARTIFACT_SCHEMA_VERSION,
+        "score_version": SCORE_VERSION,
+        "weights": _weights("fear_greed"),
+        "provenance": {
+            "validation_method": "purged_expanding_window",
+            "label_horizon_days": 365,
+            "weight_scope": "full_history_fit",
+            "training_date_range": {"start": "2018-01-01", "end": "2024-01-01"},
+            "trained_at": "2026-07-01T00:00:00+00:00",
+        },
+        "cv_results": {"folds": []},
+    }
+    if with_folds:
+        artifact["cv_results"]["folds"] = [
+            {
+                "fold": 1,
+                "weights": _weights("drawdown"),
+                "date_ranges": {"validation": "2020-01-01 to 2020-12-31"},
+            },
+            {
+                "fold": 2,
+                "weights": _weights("nupl"),
+                "date_ranges": {"validation": "2021-01-01 to 2021-12-31"},
+            },
+        ]
+    return artifact
+
+
+def test_ml_backtest_plan_prefers_fold_weights_and_marks_them_oos():
+    plan = engine._build_ml_backtest_plan(_artifact(with_folds=True))
+
+    weights, fold = engine._weights_for_backtest_date("2021-06-01", plan)
+
+    assert weights == _weights("nupl")
+    assert fold == 2
+    assert plan["evaluation_scope"] == "out_of_sample_validation_folds"
+    assert plan["is_out_of_sample"] is True
+    assert plan["weighting_source"] == "fold_specific_weights"
+    assert engine._weights_for_backtest_date("2019-12-31", plan) == (None, None)
+
+
+def test_full_history_weights_are_explicitly_not_oos():
+    plan = engine._build_ml_backtest_plan(_artifact(with_folds=False))
+
+    weights, fold = engine._weights_for_backtest_date("2021-06-01", plan)
+
+    assert weights == _weights("fear_greed")
+    assert fold is None
+    assert plan["evaluation_scope"] == "in_sample_full_history_weights"
+    assert plan["is_out_of_sample"] is False
+    assert plan["weighting_source"] == "final_full_history_weights"