bizzle/btc-accumulation-monitor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: remove leakage from legacy ML evaluation

Browse Source
This commit is contained in:
Hermes Agent
2026-07-26 22:59:21 +00:00
parent aef714d6c7
commit 81654b5743
6 changed files with 240 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ml_engine/train_and_backtest.py
+104 -24
View File
@@ -190,30 +190,22 @@ def create_accumulation_target(df, config):
fwd[i] = (close[i + period] - close[i]) / close[i] * 100
forward_returns.append(fwd)
# Rank each forward return (percentile rank, 0-1)
# Higher rank = better buy point (higher future return)
ranked = []
for fwd in forward_returns:
valid_mask = ~np.isnan(fwd)
ranks = np.full(n, np.nan)
valid_vals = fwd[valid_mask]
if len(valid_vals) > 0:
from scipy.stats import rankdata
r = rankdata(valid_vals, method="average") / len(valid_vals)
ranks[valid_mask] = r
ranked.append(ranks)
# Convert each return to a deterministic 0-100 quality score. Global
# percentile ranks leak the distribution of future validation/test rows into
# earlier training labels; a fixed tanh transform is invariant to rows added
# outside the observation's own forward horizons.
scales = tgt.get("return_scales_pct", [10.0, 30.0, 60.0])
if len(scales) != len(forward_periods) or any(scale <= 0 for scale in scales):
raise ValueError("target.return_scales_pct must contain one positive scale per forward period")
# Weighted combination of ranks -> accumulation score (0-100)
score = np.zeros(n)
valid = np.ones(n, dtype=bool)
for r, w in zip(ranked, weights):
nan_mask = np.isnan(r)
for fwd, weight, scale in zip(forward_returns, weights, scales):
nan_mask = np.isnan(fwd)
valid &= ~nan_mask
r_filled = np.where(nan_mask, 0, r)
score += w * r_filled
quality = 50.0 + 50.0 * np.tanh(np.where(nan_mask, 0.0, fwd) / scale)
score += weight * quality
# Scale to 0-100
score = score * 100
score[~valid] = np.nan
return pd.Series(score, index=df.index, name="target")
@@ -488,6 +480,27 @@ def apply_scaling_pca(X_train, X_val, X_test, config):
return X_train, X_val, X_test, scaler, pca
# ---------------------------------------------------------------------------
# Walk-forward split helpers
# ---------------------------------------------------------------------------
def _max_forward_horizon(config):
"""Return the longest forward-label horizon in candles."""
target = config.get("target", {})
key = "forward_periods_1h" if config.get("timeframe", "4h") == "1h" else "forward_periods_4h"
periods = target.get(key, [168, 720, 2160] if key.endswith("1h") else [42, 180, 540])
return max(int(period) for period in periods)
def _purge_label_overlap(frame, horizon):
"""Remove rows whose forward-return labels cross the next split boundary."""
if horizon <= 0:
return frame
if len(frame) <= horizon:
return frame.iloc[0:0]
return frame.iloc[:-horizon]
# ---------------------------------------------------------------------------
# Rolling Window Validation
# ---------------------------------------------------------------------------
@@ -499,6 +512,7 @@ def rolling_window_train_test(df, feature_cols, config):
test_size = training_cfg.get("rolling_test_size", 300)
val_pct = training_cfg.get("validation_pct", 0.15)
model_type = config.get("model_type", "xgboost")
purge_horizon = _max_forward_horizon(config)
n = len(df)
all_predictions = [] # list of (predicted_score, actual_score, close_price)
@@ -527,10 +541,16 @@ def rolling_window_train_test(df, feature_cols, config):
start += test_size
continue
# Split train into train/val
# Purge labels whose longest forward-return horizon overlaps the next
# split. Without this embargo, training and validation targets consume
# prices from the following validation/test partition.
val_split = int(len(train_full) * (1.0 - val_pct))
train_df = train_full.iloc[:val_split]
val_df = train_full.iloc[val_split:]
train_df = _purge_label_overlap(train_full.iloc[:val_split], purge_horizon)
val_df = _purge_label_overlap(train_full.iloc[val_split:], purge_horizon)
if len(train_df) < 10 or len(val_df) < 1:
start += test_size
continue
X_train = train_df[feature_cols].values
y_train = train_df["target"].values
@@ -632,6 +652,7 @@ def walk_forward_train_test(df, feature_cols, config):
n_windows = training_cfg.get("walk_forward_windows", 5)
train_pct = training_cfg.get("train_pct", 0.7)
val_pct = training_cfg.get("validation_pct", 0.15)
purge_horizon = _max_forward_horizon(config)
n = len(df)
window_size = n // n_windows
@@ -655,8 +676,8 @@ def walk_forward_train_test(df, feature_cols, config):
train_end = int(wn * train_pct)
val_end = int(wn * (train_pct + val_pct))
train_df = window_data.iloc[:train_end]
val_df = window_data.iloc[train_end:val_end]
train_df = _purge_label_overlap(window_data.iloc[:train_end], purge_horizon)
val_df = _purge_label_overlap(window_data.iloc[train_end:val_end], purge_horizon)
test_df = window_data.iloc[val_end:]
if len(test_df) < 10:
@@ -842,6 +863,46 @@ def _extract_feature_importances(model, n_features):
# Results Compilation
# ---------------------------------------------------------------------------
def simulate_periodic_accumulation(predicted_scores, close_prices, buy_threshold, contribution=1.0):
"""Compare DCA and signal strategies with equal periodic contributions.
Both strategies receive the same cash on every observation. DCA invests the
contribution immediately; the signal strategy retains cash until a buy
signal, then deploys its available balance. Terminal wealth includes cash.
"""
scores = np.asarray(predicted_scores, dtype=float)
prices = np.asarray(close_prices, dtype=float)
if len(scores) != len(prices):
raise ValueError("predicted_scores and close_prices must have equal length")
if len(prices) == 0 or contribution <= 0 or np.any(prices <= 0):
raise ValueError("prices must be positive and contribution must be greater than zero")
dca_btc = float(np.sum(contribution / prices))
model_btc = 0.0
model_cash = 0.0
for score, price in zip(scores, prices):
model_cash += contribution
if score >= buy_threshold:
model_btc += model_cash / price
model_cash = 0.0
contributed = float(len(prices) * contribution)
terminal_price = float(prices[-1])
dca_terminal = dca_btc * terminal_price
model_terminal = model_btc * terminal_price + model_cash
improvement = (model_terminal - dca_terminal) / dca_terminal * 100 if dca_terminal else 0.0
return {
"dca_contributed": contributed,
"model_contributed": contributed,
"dca_btc": dca_btc,
"model_btc": model_btc,
"model_cash": model_cash,
"dca_terminal_value": dca_terminal,
"model_terminal_value": model_terminal,
"terminal_wealth_improvement_pct": improvement,
}
def compile_results(predictions, per_window_cost_improvement,
fi_sum, fi_count, feature_cols, config):
"""Compile accumulation signal results into output JSON."""
@@ -896,6 +957,13 @@ def compile_results(predictions, per_window_cost_improvement,
model_avg = dca_avg
cost_basis_improvement = 0.0
portfolio = simulate_periodic_accumulation(
pred_scores,
close_prices,
buy_threshold=good_threshold,
contribution=1.0,
)
# --- Signal Frequency ---
signal_frequency = strong_buy_count / total_candles * 100 if total_candles > 0 else 0
@@ -946,7 +1014,14 @@ def compile_results(predictions, per_window_cost_improvement,
quality_good = False
return {
# Retained for backward compatibility; model selection uses the equal-
# capital terminal wealth metric below.
"cost_basis_improvement_pct": round(cost_basis_improvement, 2),
"terminal_wealth_improvement_pct": round(portfolio["terminal_wealth_improvement_pct"], 2),
"model_terminal_value": round(portfolio["model_terminal_value"], 6),
"dca_terminal_value": round(portfolio["dca_terminal_value"], 6),
"model_cash": round(portfolio["model_cash"], 6),
"backtest_objective": "equal_periodic_contribution_terminal_wealth",
"avg_cost_basis_model": round(model_avg, 2),
"avg_cost_basis_dca": round(dca_avg, 2),
"strong_buy_signal_count": strong_buy_count,
@@ -968,6 +1043,11 @@ def compile_results(predictions, per_window_cost_improvement,
def _empty_results(per_window):
return {
"cost_basis_improvement_pct": 0.0,
"terminal_wealth_improvement_pct": 0.0,
"model_terminal_value": 0.0,
"dca_terminal_value": 0.0,
"model_cash": 0.0,
"backtest_objective": "equal_periodic_contribution_terminal_wealth",
"avg_cost_basis_model": 0.0,
"avg_cost_basis_dca": 0.0,
"strong_buy_signal_count": 0,