bizzle/btc-accumulation-monitor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: purge ML validation label leakage

Browse Source
This commit is contained in:
Hermes Agent
2026-06-29 00:09:26 +00:00
parent 8fca6181d5
commit de2cd512cd
5 changed files with 405 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ml/optimizer.py
+228 -134
View File
@@ -43,6 +43,9 @@ START_DATE = "2018-02-01"
TRAIN_CUTOFF_DAYS = 365
# Target: forward 365d return > 30% = "good time to buy"
GOOD_BUY_THRESHOLD = 30.0
# Validation embargo/purge horizon: labels use 365-day forward returns.
LABEL_HORIZON_DAYS = 365
VALIDATION_SPLITS = 5
# The 8 core metrics we score
METRIC_KEYS = [
@@ -95,6 +98,112 @@ def score_range(value, ranges):
return 0
SCORE_KEYS = [
"puell_multiple", "mvrv_zscore", "reserve_risk", "rhodl_ratio",
"nupl", "fear_greed", "drawdown", "pct_above_200w_sma", "pct_above_lth_rp",
]
SCORE_FEATURES = [f"score_{k}" for k in SCORE_KEYS]
RAW_FEATURES = [
"raw_puell_multiple", "raw_mvrv_zscore", "raw_reserve_risk",
"raw_rhodl_ratio", "raw_nupl", "raw_fear_greed",
"raw_pct_above_200w_sma", "raw_pct_above_lth_rp", "raw_drawdown",
]
DELTA_FEATURES = [
"delta_30d_mvrv_zscore", "delta_30d_nupl",
"delta_30d_puell_multiple", "delta_30d_reserve_risk",
]
INTERACTION_FEATURES = ["mvrv_x_nupl", "puell_x_reserve"]
CYCLE_FEATURES = ["days_since_ath"]
FEATURE_COLS = SCORE_FEATURES + RAW_FEATURES + DELTA_FEATURES + INTERACTION_FEATURES + CYCLE_FEATURES
BRACKETS = [
(0, 20, "Extreme Caution"),
(21, 40, "Caution"),
(41, 55, "Neutral"),
(56, 70, "Moderate Opportunity"),
(71, 85, "Strong Accumulation"),
(86, 100, "Extreme Accumulation"),
]
def _row_date(row):
return datetime.strptime(row["date"], "%Y-%m-%d")
def purged_time_series_splits(rows, n_splits=VALIDATION_SPLITS,
label_horizon_days=LABEL_HORIZON_DAYS,
embargo_days=0):
"""Yield expanding-window splits with overlapping forward-label windows removed.
A row dated T with a 365-day forward-return label consumes information up to
T+365. For validation beginning at V, any training row whose label window
reaches V is removed. This keeps validation metrics out-of-sample for the
forward-return label, not just for features.
"""
base_splitter = TimeSeriesSplit(n_splits=n_splits)
row_dates = [_row_date(r) for r in rows]
horizon = timedelta(days=label_horizon_days)
embargo = timedelta(days=embargo_days)
for train_idx, val_idx in base_splitter.split(np.arange(len(rows))):
val_start = row_dates[val_idx[0]]
val_end = row_dates[val_idx[-1]]
purged_train = []
for idx in train_idx:
label_end = row_dates[idx] + horizon
before_validation_label_window = label_end <= val_start - embargo
after_validation_embargo = row_dates[idx] > val_end + embargo
if before_validation_label_window or after_validation_embargo:
purged_train.append(idx)
if purged_train:
yield np.array(purged_train, dtype=int), np.array(val_idx, dtype=int)
def _build_model():
return GradientBoostingClassifier(
n_estimators=300,
learning_rate=0.05,
max_depth=4,
subsample=0.8,
min_samples_leaf=20,
random_state=42,
)
def derive_metric_weights(feature_cols, importances):
"""Aggregate feature importances back to transparent score metric weights."""
metric_names = list(SCORE_KEYS)
feature_to_metric = {}
for m in metric_names:
feature_to_metric[f"score_{m}"] = m
feature_to_metric[f"raw_{m}"] = m
feature_to_metric["delta_30d_mvrv_zscore"] = "mvrv_zscore"
feature_to_metric["delta_30d_nupl"] = "nupl"
feature_to_metric["delta_30d_puell_multiple"] = "puell_multiple"
feature_to_metric["delta_30d_reserve_risk"] = "reserve_risk"
metric_importances = {m: 0.0 for m in metric_names}
for name, imp in zip(feature_cols, importances):
if name in feature_to_metric:
metric_importances[feature_to_metric[name]] += float(imp)
elif name == "mvrv_x_nupl":
metric_importances["mvrv_zscore"] += float(imp) / 2
metric_importances["nupl"] += float(imp) / 2
elif name == "puell_x_reserve":
metric_importances["puell_multiple"] += float(imp) / 2
metric_importances["reserve_risk"] += float(imp) / 2
elif name == "days_since_ath":
metric_importances["drawdown"] += float(imp)
total_imp = sum(metric_importances.values())
if total_imp > 0:
weights = {k: round(v / total_imp, 4) for k, v in metric_importances.items()}
else:
weights = {k: round(1 / len(metric_importances), 4) for k in metric_importances}
return dict(sorted(weights.items(), key=lambda x: x[1], reverse=True))
def build_dataset(index, thresholds):
"""Build aligned training dataset: metric scores + forward returns."""
# Get all dates from 2018-02-01 onward
@@ -257,39 +366,33 @@ def train_model(rows):
log.info("Target distribution: %d positive (%.1f%%), %d negative",
positive, positive / len(labeled) * 100, len(labeled) - positive)
# Feature columns: scores + raw values + deltas + interactions + cycle position
score_features = [
"score_puell_multiple", "score_mvrv_zscore", "score_reserve_risk",
"score_rhodl_ratio", "score_nupl", "score_fear_greed",
"score_drawdown", "score_pct_above_200w_sma", "score_pct_above_lth_rp",
]
raw_features = [
"raw_puell_multiple", "raw_mvrv_zscore", "raw_reserve_risk",
"raw_rhodl_ratio", "raw_nupl", "raw_fear_greed",
"raw_pct_above_200w_sma", "raw_pct_above_lth_rp", "raw_drawdown",
]
delta_features = [
"delta_30d_mvrv_zscore", "delta_30d_nupl",
"delta_30d_puell_multiple", "delta_30d_reserve_risk",
]
interaction_features = ["mvrv_x_nupl", "puell_x_reserve"]
cycle_features = ["days_since_ath"]
feature_cols = score_features + raw_features + delta_features + interaction_features + cycle_features
feature_cols = FEATURE_COLS
X = np.array([[r[f] for f in feature_cols] for r in labeled])
y = np.array([r["target"] for r in labeled])
log.info("Feature matrix: %d samples x %d features", X.shape[0], X.shape[1])
# Time-series cross-validation (expanding window, 5 splits)
tscv = TimeSeriesSplit(n_splits=5)
# Purged time-series cross-validation. Standard TimeSeriesSplit is not
# enough here because each label consumes the next 365 days of returns.
cv_scores = []
cv_f1 = []
cv_precision = []
cv_recall = []
fold_results = []
for fold, (train_idx, val_idx) in enumerate(tscv.split(X)):
splits = list(purged_time_series_splits(
labeled,
n_splits=VALIDATION_SPLITS,
label_horizon_days=LABEL_HORIZON_DAYS,
embargo_days=0,
))
if not splits:
log.error("No viable purged validation splits. Need more history for %dd label horizon.",
LABEL_HORIZON_DAYS)
return None
for fold, (train_idx, val_idx) in enumerate(splits):
X_train, X_val = X[train_idx], X[val_idx]
y_train, y_val = y[train_idx], y[val_idx]
@@ -297,14 +400,7 @@ def train_model(rows):
X_train_s = scaler.fit_transform(X_train)
X_val_s = scaler.transform(X_val)
model = GradientBoostingClassifier(
n_estimators=300,
learning_rate=0.05,
max_depth=4,
subsample=0.8,
min_samples_leaf=20,
random_state=42,
)
model = _build_model()
model.fit(X_train_s, y_train)
y_pred = model.predict(X_val_s)
@@ -320,27 +416,40 @@ def train_model(rows):
cv_precision.append(prec)
cv_recall.append(rec)
train_dates = f"{labeled[train_idx[0]]['date']} to {labeled[train_idx[-1]]['date']}"
val_dates = f"{labeled[val_idx[0]]['date']} to {labeled[val_idx[-1]]['date']}"
fold_weights = derive_metric_weights(feature_cols, model.feature_importances_)
fold_results.append({
"fold": fold + 1,
"train_idx": train_idx.tolist(),
"val_idx": val_idx.tolist(),
"weights": fold_weights,
"metrics": {
"auc": round(float(auc), 4),
"f1": round(float(f1), 4),
"precision": round(float(prec), 4),
"recall": round(float(rec), 4),
},
"date_ranges": {
"train": f"{labeled[train_idx[0]]['date']} to {labeled[train_idx[-1]]['date']}",
"validation": f"{labeled[val_idx[0]]['date']} to {labeled[val_idx[-1]]['date']}",
},
"n_train": len(train_idx),
"n_validation": len(val_idx),
})
train_dates = fold_results[-1]["date_ranges"]["train"]
val_dates = fold_results[-1]["date_ranges"]["validation"]
log.info("Fold %d: Train %s | Val %s | AUC=%.3f F1=%.3f P=%.3f R=%.3f",
fold + 1, train_dates, val_dates, auc, f1, prec, rec)
log.info("CV Mean AUC: %.3f (+/- %.3f)", np.mean(cv_scores), np.std(cv_scores))
log.info("CV Mean F1: %.3f (+/- %.3f)", np.mean(cv_f1), np.std(cv_f1))
log.info("Purged CV Mean AUC: %.3f (+/- %.3f)", np.mean(cv_scores), np.std(cv_scores))
log.info("Purged CV Mean F1: %.3f (+/- %.3f)", np.mean(cv_f1), np.std(cv_f1))
# Train final model on all labeled data
log.info("Training final model on all %d labeled samples...", len(labeled))
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
final_model = GradientBoostingClassifier(
n_estimators=300,
learning_rate=0.05,
max_depth=4,
subsample=0.8,
min_samples_leaf=20,
random_state=42,
)
final_model = _build_model()
final_model.fit(X_scaled, y)
# Feature importances
@@ -357,48 +466,7 @@ def train_model(rows):
bar = "#" * int(imp * 200)
log.info(" %-30s %.4f %s", name, imp, bar)
# Extract optimal weights by aggregating importance per metric
# Map each feature back to its parent metric
metric_names = [
"puell_multiple", "mvrv_zscore", "reserve_risk", "rhodl_ratio",
"nupl", "fear_greed", "drawdown", "pct_above_200w_sma", "pct_above_lth_rp",
]
feature_to_metric = {}
for m in metric_names:
feature_to_metric[f"score_{m}"] = m
feature_to_metric[f"raw_{m}"] = m
# Delta features map to their base metric
feature_to_metric["delta_30d_mvrv_zscore"] = "mvrv_zscore"
feature_to_metric["delta_30d_nupl"] = "nupl"
feature_to_metric["delta_30d_puell_multiple"] = "puell_multiple"
feature_to_metric["delta_30d_reserve_risk"] = "reserve_risk"
# Interaction terms split evenly between constituent metrics
# mvrv_x_nupl -> mvrv_zscore + nupl
# puell_x_reserve -> puell_multiple + reserve_risk
metric_importances = {m: 0.0 for m in metric_names}
for name, imp in feat_imp:
if name in feature_to_metric:
metric_importances[feature_to_metric[name]] += imp
elif name == "mvrv_x_nupl":
metric_importances["mvrv_zscore"] += imp / 2
metric_importances["nupl"] += imp / 2
elif name == "puell_x_reserve":
metric_importances["puell_multiple"] += imp / 2
metric_importances["reserve_risk"] += imp / 2
# days_since_ath maps to drawdown conceptually
elif name == "days_since_ath":
metric_importances["drawdown"] += imp
# Normalize weights to sum to 1
total_imp = sum(metric_importances.values())
if total_imp > 0:
weights = {k: round(v / total_imp, 4) for k, v in metric_importances.items()}
else:
weights = {k: round(1 / len(metric_importances), 4) for k in metric_importances}
# Sort by weight descending
weights = dict(sorted(weights.items(), key=lambda x: x[1], reverse=True))
weights = derive_metric_weights(feature_cols, importances)
log.info("\nOptimal Metric Weights:")
log.info("-" * 50)
@@ -413,6 +481,7 @@ def train_model(rows):
log.info("COMPARISON BACKTEST: ML-Weighted vs Equal-Weight")
log.info("=" * 60)
comparison = run_comparison(rows, weights)
out_of_sample_comparison = run_out_of_sample_comparison(labeled, fold_results)
# Build output
result = {
@@ -424,6 +493,9 @@ def train_model(rows):
"mean_f1": round(float(np.mean(cv_f1)), 4),
"mean_precision": round(float(np.mean(cv_precision)), 4),
"mean_recall": round(float(np.mean(cv_recall)), 4),
"validation_method": "purged_expanding_window",
"label_horizon_days": LABEL_HORIZON_DAYS,
"folds": fold_results,
},
"training_info": {
"n_samples": len(labeled),
@@ -435,66 +507,47 @@ def train_model(rows):
"model": "GradientBoostingClassifier",
},
"comparison": comparison,
"out_of_sample_comparison": out_of_sample_comparison,
"trained_at": datetime.now(tz=__import__('datetime').timezone.utc).isoformat(),
}
return result
def run_comparison(rows, ml_weights):
"""Compare ML-weighted scoring vs equal-weight scoring across score brackets."""
# Metrics used in scoring (maps to score_* columns)
score_keys = [
"puell_multiple", "mvrv_zscore", "reserve_risk", "rhodl_ratio",
"nupl", "fear_greed", "drawdown", "pct_above_200w_sma", "pct_above_lth_rp",
]
n_metrics = len(score_keys)
equal_weight = 1.0 / n_metrics
def _composite_score(row, mode, ml_weights=None):
scores = [row[f"score_{k}"] for k in SCORE_KEYS]
if mode == "equal_weight" or not ml_weights:
return sum(scores) / len(SCORE_KEYS) * 10
equal_weight = 1.0 / len(SCORE_KEYS)
weighted_sum = sum(row[f"score_{k}"] * ml_weights.get(k, equal_weight) for k in SCORE_KEYS)
return weighted_sum * 10
brackets = [
(0, 20, "Extreme Caution"),
(21, 40, "Caution"),
(41, 55, "Neutral"),
(56, 70, "Moderate Opportunity"),
(71, 85, "Strong Accumulation"),
(86, 100, "Extreme Accumulation"),
]
# Only use rows with forward returns
scored_rows = [r for r in rows if "fwd_365d" in r]
results = {"equal_weight": [], "ml_weighted": []}
for mode in ["equal_weight", "ml_weighted"]:
for r in scored_rows:
scores = [r[f"score_{k}"] for k in score_keys]
if mode == "equal_weight":
composite = sum(scores) / n_metrics * 10
else:
weighted_sum = sum(r[f"score_{k}"] * ml_weights.get(k, equal_weight) for k in score_keys)
composite = weighted_sum * 10
r[f"composite_{mode}"] = composite
for low, high, label in brackets:
days_in = [r for r in scored_rows if low <= r[f"composite_{mode}"] <= high]
if not days_in:
results[mode].append({
"range": f"{low}-{high}", "label": label,
"days": 0, "avg_365d": None,
})
continue
returns_365 = [r["fwd_365d"] for r in days_in]
win_rate = len([r for r in returns_365 if r > 0]) / len(returns_365) * 100
results[mode].append({
"range": f"{low}-{high}",
"label": label,
"days": len(days_in),
"avg_365d": round(sum(returns_365) / len(returns_365), 2),
"median_365d": round(sorted(returns_365)[len(returns_365) // 2], 2),
"win_rate_365d": round(win_rate, 1),
def _summarize_brackets(scored_rows, score_key):
results = []
for low, high, label in BRACKETS:
days_in = [r for r in scored_rows if low <= r[score_key] <= high]
if not days_in:
results.append({
"range": f"{low}-{high}", "label": label,
"days": 0, "avg_365d": None,
})
continue
returns_365 = [r["fwd_365d"] for r in days_in]
returns_sorted = sorted(returns_365)
win_rate = len([r for r in returns_365 if r > 0]) / len(returns_365) * 100
results.append({
"range": f"{low}-{high}",
"label": label,
"days": len(days_in),
"avg_365d": round(sum(returns_365) / len(returns_365), 2),
"median_365d": round(returns_sorted[len(returns_sorted) // 2], 2),
"win_rate_365d": round(win_rate, 1),
})
return results
# Print comparison
def _log_comparison_table(results):
log.info("\n%-18s | %-8s %-8s %-8s | %-8s %-8s %-8s",
"Bracket", "EQ Avg", "EQ Med", "EQ Win%", "ML Avg", "ML Med", "ML Win%")
log.info("-" * 80)
@@ -508,6 +561,47 @@ def run_comparison(rows, ml_weights):
log.info("%-18s | %-8s %-8s %-8s | %-8s %-8s %-8s",
eq["label"], eq_avg, eq_med, eq_win, ml_avg, ml_med, ml_win)
def run_comparison(rows, ml_weights):
"""Compare final ML-weighted scoring vs equal-weight scoring across all labeled rows.
This is retained for backwards compatibility with existing output. It is an
in-sample/full-history comparison; prefer out_of_sample_comparison for model
selection decisions.
"""
scored_rows = [dict(r) for r in rows if "fwd_365d" in r]
for r in scored_rows:
r["composite_equal_weight"] = _composite_score(r, "equal_weight")
r["composite_ml_weighted"] = _composite_score(r, "ml_weighted", ml_weights)
results = {
"equal_weight": _summarize_brackets(scored_rows, "composite_equal_weight"),
"ml_weighted": _summarize_brackets(scored_rows, "composite_ml_weighted"),
}
_log_comparison_table(results)
return results
def run_out_of_sample_comparison(rows, fold_results):
"""Compare fold-specific ML weights on validation rows only."""
validation_rows = []
for fold in fold_results:
weights = fold.get("weights", {})
for idx in fold.get("val_idx", []):
if idx >= len(rows) or "fwd_365d" not in rows[idx]:
continue
r = dict(rows[idx])
r["fold"] = fold.get("fold")
r["composite_equal_weight"] = _composite_score(r, "equal_weight")
r["composite_ml_weighted"] = _composite_score(r, "ml_weighted", weights)
validation_rows.append(r)
results = {
"folds": len(fold_results),
"validation_days": len(validation_rows),
"equal_weight": _summarize_brackets(validation_rows, "composite_equal_weight"),
"ml_weighted": _summarize_brackets(validation_rows, "composite_ml_weighted"),
}
return results