diff --git a/backtesting/engine.py b/backtesting/engine.py index 6b1dd86..859a2a9 100644 --- a/backtesting/engine.py +++ b/backtesting/engine.py @@ -8,6 +8,7 @@ from collections import defaultdict from datetime import datetime, timedelta from scoring.policy import SCORE_BRACKETS, SCORE_VERSION, score_in_bracket +from ml.artifacts import validate_ml_artifact log = logging.getLogger(__name__) @@ -117,11 +118,15 @@ def _compute_ath_series(price_lookup, dates): def _load_ml_weights(): - """Load ML weights for ML-optimized scoring mode.""" + """Load only schema/provenance-valid ML weights.""" ml_path = _os.path.join(_os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))), "config", "ml_weights.json") try: with open(ml_path) as f: data = _json.load(f) + status = validate_ml_artifact(data) + if not status["valid"]: + log.error("Rejected invalid ML artifact: %s", ", ".join(status["errors"])) + return {} return data.get("weights", {}) except Exception: return {} diff --git a/ml/artifacts.py b/ml/artifacts.py new file mode 100644 index 0000000..4134c73 --- /dev/null +++ b/ml/artifacts.py @@ -0,0 +1,91 @@ +"""Validation for persisted ML scoring artifacts.""" + +import math + +from scoring.policy import SCORE_VERSION + +ML_ARTIFACT_SCHEMA_VERSION = 2 +REQUIRED_WEIGHT_KEYS = frozenset({ + "puell_multiple", + "mvrv_zscore", + "reserve_risk", + "rhodl_ratio", + "nupl", + "fear_greed", + "drawdown", + "pct_above_200w_sma", + "pct_above_lth_rp", +}) + + +def _weights_valid(weights): + if not isinstance(weights, dict) or not REQUIRED_WEIGHT_KEYS.issubset(weights): + return False + values = [weights[key] for key in REQUIRED_WEIGHT_KEYS] + return all( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and value >= 0 + for value in values + ) and sum(values) > 0 + + +def _has_oos_fold_weights(artifact): + folds = artifact.get("cv_results", {}).get("folds", []) + if not isinstance(folds, list) or not folds: + return False + for fold in folds: + validation_range = fold.get("date_ranges", {}).get("validation") + if not validation_range or not _weights_valid(fold.get("weights")): + return False + return True + + +def validate_ml_artifact(artifact): + """Return machine-readable validity and provenance for an ML artifact.""" + errors = [] + if not isinstance(artifact, dict): + artifact = {} + errors.append("artifact_object") + + schema_version = artifact.get("artifact_schema_version") + if schema_version != ML_ARTIFACT_SCHEMA_VERSION: + errors.append("artifact_schema_version") + + score_version = artifact.get("score_version") + if score_version != SCORE_VERSION: + errors.append("score_version") + + if not _weights_valid(artifact.get("weights")): + errors.append("weights") + + provenance = artifact.get("provenance") + if not isinstance(provenance, dict): + provenance = {} + errors.append("provenance") + else: + required_provenance = { + "validation_method", + "label_horizon_days", + "weight_scope", + "training_date_range", + "trained_at", + } + if not required_provenance.issubset(provenance): + errors.append("provenance") + if provenance.get("validation_method") != "purged_expanding_window": + errors.append("purged_validation") + if provenance.get("label_horizon_days") != 365: + errors.append("label_horizon_days") + if provenance.get("weight_scope") != "full_history_fit": + errors.append("weight_scope") + + return { + "valid": not errors, + "schema_version": schema_version, + "score_version": score_version, + "weight_scope": provenance.get("weight_scope"), + "has_oos_fold_weights": _has_oos_fold_weights(artifact), + "errors": list(dict.fromkeys(errors)), + } diff --git a/ml/optimizer.py b/ml/optimizer.py index 67c0ca5..622b0a8 100644 --- a/ml/optimizer.py +++ b/ml/optimizer.py @@ -26,7 +26,8 @@ from sklearn.metrics import ( from sklearn.model_selection import TimeSeriesSplit from sklearn.preprocessing import StandardScaler -from scoring.policy import SCORE_BRACKETS, score_in_bracket +from scoring.policy import SCORE_BRACKETS, SCORE_VERSION, score_in_bracket +from ml.artifacts import ML_ARTIFACT_SCHEMA_VERSION logging.basicConfig( level=logging.INFO, @@ -478,8 +479,12 @@ def train_model(rows): comparison = run_comparison(rows, weights) out_of_sample_comparison = run_out_of_sample_comparison(labeled, fold_results) - # Build output + # Build output. Final weights are fitted on all labeled history for live use; + # only the fold weights below are valid for OOS comparisons. + trained_at = datetime.now(tz=__import__('datetime').timezone.utc).isoformat() result = { + "artifact_schema_version": ML_ARTIFACT_SCHEMA_VERSION, + "score_version": SCORE_VERSION, "weights": weights, "feature_importances": {name: round(float(imp), 6) for name, imp in feat_imp}, "cv_results": { @@ -501,9 +506,19 @@ def train_model(rows): "date_range": f"{labeled[0]['date']} to {labeled[-1]['date']}", "model": "GradientBoostingClassifier", }, + "provenance": { + "validation_method": "purged_expanding_window", + "label_horizon_days": LABEL_HORIZON_DAYS, + "weight_scope": "full_history_fit", + "training_date_range": { + "start": labeled[0]["date"], + "end": labeled[-1]["date"], + }, + "trained_at": trained_at, + }, "comparison": comparison, "out_of_sample_comparison": out_of_sample_comparison, - "trained_at": datetime.now(tz=__import__('datetime').timezone.utc).isoformat(), + "trained_at": trained_at, } return result diff --git a/scoring/engine.py b/scoring/engine.py index c8ea9d5..1f00c22 100644 --- a/scoring/engine.py +++ b/scoring/engine.py @@ -5,6 +5,7 @@ import os import logging from scoring.policy import SCORE_VERSION, assessment_for_score +from ml.artifacts import validate_ml_artifact log = logging.getLogger(__name__) @@ -580,16 +581,30 @@ _ML_KEY_MAP = { } +_ml_artifact_status = {"valid": False, "errors": ["not_loaded"]} + + def load_ml_weights(): - """Load ML-optimized weights from config.""" + """Load weights only when their schema and training provenance are valid.""" + global _ml_artifact_status try: with open(ML_WEIGHTS_PATH) as f: data = json.load(f) + _ml_artifact_status = validate_ml_artifact(data) + if not _ml_artifact_status["valid"]: + log.error("Rejected invalid ML artifact: %s", ", ".join(_ml_artifact_status["errors"])) + return {} return data.get("weights", {}) - except Exception: + except Exception as exc: + _ml_artifact_status = {"valid": False, "errors": [f"load_error:{exc}"]} return {} +def get_ml_artifact_status(): + """Return the status from the most recent artifact load attempt.""" + return dict(_ml_artifact_status) + + def score_all_ml(metrics): """Score all metrics using ML-optimized weights. @@ -604,7 +619,12 @@ def score_all_ml(metrics): if not ml_weights: # Fallback to classic if no ML weights available classic["ml_mode"] = False - classic["ml_error"] = "ML weights not found — run ml/optimizer.py" + status = get_ml_artifact_status() + if status.get("errors") and status["errors"] != ["not_loaded"]: + classic["ml_error"] = "ML artifact invalid: " + ", ".join(status["errors"]) + else: + classic["ml_error"] = "ML weights not found — run ml/optimizer.py" + classic["ml_artifact"] = status return classic results = classic["metrics"] diff --git a/tests/test_ml_artifacts.py b/tests/test_ml_artifacts.py new file mode 100644 index 0000000..853d0dc --- /dev/null +++ b/tests/test_ml_artifacts.py @@ -0,0 +1,58 @@ +import json +from pathlib import Path + +from ml import artifacts +from scoring import engine + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _valid_artifact(): + return { + "artifact_schema_version": artifacts.ML_ARTIFACT_SCHEMA_VERSION, + "score_version": artifacts.SCORE_VERSION, + "weights": {key: 1 / len(artifacts.REQUIRED_WEIGHT_KEYS) for key in artifacts.REQUIRED_WEIGHT_KEYS}, + "provenance": { + "validation_method": "purged_expanding_window", + "label_horizon_days": 365, + "weight_scope": "full_history_fit", + "training_date_range": {"start": "2018-02-01", "end": "2025-03-21"}, + "trained_at": "2026-07-01T00:00:00+00:00", + }, + } + + +def test_pre_purge_repository_artifact_is_rejected_with_actionable_status(): + artifact_path = REPO_ROOT / "config" / "ml_weights.json" + artifact = json.loads(artifact_path.read_text()) + + status = artifacts.validate_ml_artifact(artifact) + + assert status["valid"] is False + assert status["schema_version"] is None + assert "artifact_schema_version" in status["errors"] + + +def test_valid_artifact_requires_schema_score_version_and_purged_provenance(): + artifact = _valid_artifact() + + status = artifacts.validate_ml_artifact(artifact) + + assert status == { + "valid": True, + "schema_version": artifacts.ML_ARTIFACT_SCHEMA_VERSION, + "score_version": artifacts.SCORE_VERSION, + "weight_scope": "full_history_fit", + "has_oos_fold_weights": False, + "errors": [], + } + + +def test_live_scoring_refuses_schema_less_weights(tmp_path, monkeypatch): + path = tmp_path / "ml_weights.json" + path.write_text(json.dumps({"weights": {"fear_greed": 1.0}})) + monkeypatch.setattr(engine, "ML_WEIGHTS_PATH", str(path)) + + assert engine.load_ml_weights() == {} + assert engine.get_ml_artifact_status()["valid"] is False diff --git a/tests/test_scoring_engine_ml.py b/tests/test_scoring_engine_ml.py index ffb4ef5..a0c17aa 100644 --- a/tests/test_scoring_engine_ml.py +++ b/tests/test_scoring_engine_ml.py @@ -59,5 +59,6 @@ def test_score_all_ml_preserves_classic_fallback_when_weights_missing(monkeypatc scored = engine.score_all_ml(_complete_metrics()) assert scored["ml_mode"] is False - assert scored["ml_error"] == "ML weights not found — run ml/optimizer.py" + assert scored["ml_error"] + assert scored["ml_artifact"]["valid"] is False assert "classic_score" not in scored