773 lines
28 KiB
Python
773 lines
28 KiB
Python
"""Historical backtest engine for Bitcoin Accumulation Zone scoring."""
|
|
|
|
import copy
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
import threading
|
|
from collections import defaultdict
|
|
from datetime import datetime, timedelta
|
|
|
|
from scoring.policy import SCORE_BRACKETS, SCORE_VERSION, score_in_bracket
|
|
from ml.artifacts import validate_ml_artifact
|
|
from backtesting.statistics import summarize_returns
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
sys.path.insert(0, BASE_DIR)
|
|
|
|
HISTORY_PATH = os.path.join(BASE_DIR, "data", "history.json")
|
|
CACHE_PATH = os.path.join(BASE_DIR, "data", "cache.json")
|
|
ML_WEIGHTS_PATH = os.path.join(BASE_DIR, "config", "ml_weights.json")
|
|
|
|
_BACKTEST_CACHE = {}
|
|
_BACKTEST_CACHE_LOCK = threading.Lock()
|
|
_BACKTEST_CACHE_LIMIT = 4
|
|
|
|
# Score brackets matching the dashboard assessment levels
|
|
BRACKETS = SCORE_BRACKETS
|
|
|
|
# Scoring thresholds — load from config/thresholds.json (single source of truth)
|
|
import os as _os
|
|
import json as _json
|
|
|
|
_THRESH_PATH = _os.path.join(_os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))), "config", "thresholds.json")
|
|
try:
|
|
with open(_THRESH_PATH) as _f:
|
|
_THRESH = _json.load(_f)
|
|
except Exception:
|
|
_THRESH = {}
|
|
|
|
METRIC_SCORERS = {
|
|
"fear_greed": {"ranges": _THRESH.get("fear_greed", {}).get("ranges", [[0, 15, 10], [15, 30, 8], [30, 45, 5], [45, 55, 3], [55, 75, 1], [75, None, 0]])},
|
|
"puell_multiple": {"ranges": _THRESH.get("puell_multiple", {}).get("ranges", [[None, 0.4, 10], [0.4, 0.7, 8], [0.7, 1.0, 5], [1.0, 1.5, 3], [1.5, 2.0, 1], [2.0, None, 0]])},
|
|
"mvrv_zscore": {"ranges": _THRESH.get("mvrv_zscore", {}).get("ranges", [[None, 0, 10], [0, 1.0, 8], [1.0, 2.0, 5], [2.0, 3.0, 3], [3.0, 5.0, 1], [5.0, None, 0]])},
|
|
"reserve_risk": {"ranges": _THRESH.get("reserve_risk", {}).get("ranges", [[None, 0.002, 10], [0.002, 0.005, 7], [0.005, 0.01, 4], [0.01, 0.02, 2], [0.02, None, 0]])},
|
|
"rhodl_ratio": {"ranges": _THRESH.get("rhodl_ratio", {}).get("ranges", [[None, 200, 10], [200, 1000, 7], [1000, 5000, 4], [5000, 20000, 1], [20000, None, 0]])},
|
|
"nupl": {"ranges": _THRESH.get("nupl", {}).get("ranges", [[None, 0, 10], [0, 0.3, 8], [0.3, 0.5, 4], [0.5, 0.75, 1], [0.75, None, 0]])},
|
|
}
|
|
|
|
RATIO_SCORERS = {
|
|
"price_vs_200w_sma": {
|
|
"ranges": _THRESH.get("price_vs_200w_sma", {}).get("ranges", [[None, 0, 10], [0, 30, 7], [30, 60, 5], [60, 100, 2], [100, None, 0]]),
|
|
"price_key": "btc_price",
|
|
"ref_key": "200w_sma",
|
|
},
|
|
"lth_realized_price": {
|
|
"ranges": _THRESH.get("lth_realized_price", {}).get("ranges", [[None, 0, 10], [0, 30, 7], [30, 80, 5], [80, 150, 3], [150, None, 1]]),
|
|
"price_key": "btc_price",
|
|
"ref_key": "lth_realized_price",
|
|
},
|
|
}
|
|
|
|
BACKTEST_METRIC_PANEL = tuple(METRIC_SCORERS) + tuple(RATIO_SCORERS) + ("drawdown",)
|
|
METRIC_MAX_AGE_DAYS = {
|
|
"fear_greed": 2,
|
|
"puell_multiple": 7,
|
|
"mvrv_zscore": 7,
|
|
"reserve_risk": 7,
|
|
"rhodl_ratio": 7,
|
|
"nupl": 7,
|
|
"btc_price": 3,
|
|
"btc_price_coingecko": 3,
|
|
"btc_price_sma": 3,
|
|
"btc_price_lth": 3,
|
|
"200w_sma": 7,
|
|
"lth_realized_price": 7,
|
|
}
|
|
|
|
DRAWDOWN_RANGES = _THRESH.get("drawdown", {}).get("ranges", [[60, None, 10], [40, 60, 8], [25, 40, 6], [15, 25, 4], [5, 15, 2], [None, 5, 0]])
|
|
|
|
|
|
def _score_range(value, ranges):
|
|
"""Score a value using range-based thresholds."""
|
|
if value is None:
|
|
return None
|
|
for low, high, score in ranges:
|
|
low_ok = low is None or value >= low
|
|
high_ok = high is None or value < high
|
|
if low_ok and high_ok:
|
|
return score
|
|
return 0
|
|
|
|
|
|
def _build_daily_index(history):
|
|
"""Build a dict mapping metric_key -> {date_str: value} for fast lookup."""
|
|
index = {}
|
|
for key, data in history.items():
|
|
if key.startswith("_") or not isinstance(data, dict) or "dates" not in data:
|
|
continue
|
|
lookup = {}
|
|
for d, v in zip(data["dates"], data["values"]):
|
|
lookup[d] = v
|
|
index[key] = lookup
|
|
return index
|
|
|
|
|
|
def _get_all_dates(index):
|
|
"""Get sorted union of all dates across all metrics."""
|
|
all_dates = set()
|
|
for lookup in index.values():
|
|
all_dates.update(lookup.keys())
|
|
return sorted(all_dates)
|
|
|
|
|
|
def _last_known_value(lookup, date, max_lookback=0):
|
|
"""Get value for date, or a prior value within an explicit lookback."""
|
|
if date in lookup:
|
|
return lookup[date]
|
|
d = datetime.strptime(date, "%Y-%m-%d")
|
|
for i in range(1, max_lookback + 1):
|
|
prev = (d - timedelta(days=i)).strftime("%Y-%m-%d")
|
|
if prev in lookup:
|
|
return lookup[prev]
|
|
return None
|
|
|
|
|
|
def _metric_observation(lookup, date, metric_key):
|
|
"""Return value, source date, and age under a metric-specific freshness rule."""
|
|
max_age = METRIC_MAX_AGE_DAYS.get(metric_key, 0)
|
|
target = datetime.strptime(date, "%Y-%m-%d")
|
|
for age in range(max_age + 1):
|
|
source_date = (target - timedelta(days=age)).strftime("%Y-%m-%d")
|
|
if source_date in lookup:
|
|
return lookup[source_date], source_date, age
|
|
return None, None, None
|
|
|
|
|
|
def _compute_ath_series(price_lookup, dates):
|
|
"""Compute running ATH and drawdown for each date."""
|
|
ath = 0
|
|
drawdowns = {}
|
|
for d in dates:
|
|
p = price_lookup.get(d)
|
|
if p is None:
|
|
continue
|
|
if p > ath:
|
|
ath = p
|
|
if ath > 0:
|
|
drawdowns[d] = ((ath - p) / ath) * 100
|
|
return drawdowns
|
|
|
|
|
|
def _load_ml_artifact():
|
|
"""Load an ML artifact and return it with validation status."""
|
|
ml_path = _os.path.join(_os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))), "config", "ml_weights.json")
|
|
try:
|
|
with open(ml_path) as f:
|
|
data = _json.load(f)
|
|
status = validate_ml_artifact(data)
|
|
if not status["valid"]:
|
|
log.error("Rejected invalid ML artifact: %s", ", ".join(status["errors"]))
|
|
return None, status
|
|
return data, status
|
|
except Exception as exc:
|
|
return None, {"valid": False, "errors": [f"load_error:{exc}"]}
|
|
|
|
|
|
def _build_ml_backtest_plan(artifact):
|
|
"""Choose OOS fold weights when available; otherwise mark final weights in-sample."""
|
|
status = validate_ml_artifact(artifact)
|
|
if not status["valid"]:
|
|
raise ValueError("invalid ML artifact: " + ", ".join(status["errors"]))
|
|
|
|
if status["has_oos_fold_weights"]:
|
|
folds = []
|
|
for fold in artifact["cv_results"]["folds"]:
|
|
start, separator, end = fold["date_ranges"]["validation"].partition(" to ")
|
|
if not separator:
|
|
raise ValueError("invalid validation date range")
|
|
folds.append({
|
|
"fold": fold.get("fold"),
|
|
"start": start,
|
|
"end": end,
|
|
"weights": fold["weights"],
|
|
})
|
|
return {
|
|
"evaluation_scope": "out_of_sample_validation_folds",
|
|
"is_out_of_sample": True,
|
|
"weighting_source": "fold_specific_weights",
|
|
"folds": folds,
|
|
"weights": None,
|
|
}
|
|
|
|
return {
|
|
"evaluation_scope": "in_sample_full_history_weights",
|
|
"is_out_of_sample": False,
|
|
"weighting_source": "final_full_history_weights",
|
|
"folds": [],
|
|
"weights": artifact["weights"],
|
|
}
|
|
|
|
|
|
def _weights_for_backtest_date(date, plan):
|
|
"""Return date-appropriate weights and fold number for an ML plan."""
|
|
if plan["is_out_of_sample"]:
|
|
for fold in plan["folds"]:
|
|
if fold["start"] <= date <= fold["end"]:
|
|
return fold["weights"], fold["fold"]
|
|
return None, None
|
|
return plan["weights"], None
|
|
|
|
|
|
def _load_ml_weights():
|
|
"""Compatibility helper returning valid final weights only."""
|
|
artifact, _ = _load_ml_artifact()
|
|
return artifact.get("weights", {}) if artifact else {}
|
|
|
|
# ML weight key mapping (backtest metric keys -> ML weight keys)
|
|
_BT_ML_KEY_MAP = {
|
|
"fear_greed": "fear_greed",
|
|
"puell_multiple": "puell_multiple",
|
|
"mvrv_zscore": "mvrv_zscore",
|
|
"reserve_risk": "reserve_risk",
|
|
"rhodl_ratio": "rhodl_ratio",
|
|
"nupl": "nupl",
|
|
"price_vs_200w_sma": "pct_above_200w_sma",
|
|
"lth_realized_price": "pct_above_lth_rp",
|
|
"drawdown": "drawdown",
|
|
}
|
|
|
|
|
|
def _common_panel_current_score(scored, ml_weights=None):
|
|
"""Recompute the current score using only metrics present historically."""
|
|
by_key = {
|
|
metric.get("key"): metric.get("score")
|
|
for metric in scored.get("metrics", [])
|
|
if metric.get("key") in BACKTEST_METRIC_PANEL and metric.get("score") is not None
|
|
}
|
|
available_keys = [key for key in BACKTEST_METRIC_PANEL if key in by_key]
|
|
coverage = {
|
|
"available_count": len(available_keys),
|
|
"panel_count": len(BACKTEST_METRIC_PANEL),
|
|
"available_keys": available_keys,
|
|
}
|
|
if not available_keys:
|
|
return None, coverage
|
|
|
|
if ml_weights:
|
|
weighted = [
|
|
(by_key[key], ml_weights.get(_BT_ML_KEY_MAP[key], 0.0))
|
|
for key in available_keys
|
|
]
|
|
weight_total = sum(weight for _, weight in weighted)
|
|
if weight_total > 0:
|
|
return round(sum(score * weight for score, weight in weighted) / weight_total * 10, 1), coverage
|
|
|
|
return round(sum(by_key[key] for key in available_keys) / len(available_keys) * 10, 1), coverage
|
|
|
|
|
|
def _backtest_data_quality_metadata(metric_counts):
|
|
"""Describe historical panel, coverage, and freshness assumptions."""
|
|
coverage = {
|
|
"minimum_metrics": min(metric_counts),
|
|
"maximum_metrics": max(metric_counts),
|
|
"average_metrics": round(sum(metric_counts) / len(metric_counts), 1),
|
|
"panel_count": len(BACKTEST_METRIC_PANEL),
|
|
} if metric_counts else {
|
|
"minimum_metrics": 0,
|
|
"maximum_metrics": 0,
|
|
"average_metrics": 0,
|
|
"panel_count": len(BACKTEST_METRIC_PANEL),
|
|
}
|
|
return {
|
|
"metric_panel": {
|
|
"id": "historical-common-v1",
|
|
"keys": list(BACKTEST_METRIC_PANEL),
|
|
"count": len(BACKTEST_METRIC_PANEL),
|
|
},
|
|
"coverage": coverage,
|
|
"staleness_days": dict(METRIC_MAX_AGE_DAYS),
|
|
}
|
|
|
|
|
|
def score_day(date, index, drawdowns, ml_weights=None):
|
|
"""Score a single day using all available metrics. Returns (composite_score, details, n_metrics).
|
|
|
|
If ml_weights is provided, uses ML-optimized weighting instead of equal weights.
|
|
details includes both "score" and "raw" (the actual metric value before scoring).
|
|
"""
|
|
scores = []
|
|
details = {}
|
|
|
|
# Simple range-based metrics
|
|
for metric_key, cfg in METRIC_SCORERS.items():
|
|
val, observed_date, age_days = _metric_observation(
|
|
index.get(metric_key, {}), date, metric_key
|
|
)
|
|
if val is not None:
|
|
s = _score_range(val, cfg["ranges"])
|
|
if s is not None:
|
|
scores.append(s)
|
|
details[metric_key] = {
|
|
"value": val,
|
|
"score": s,
|
|
"raw": val,
|
|
"observed_date": observed_date,
|
|
"age_days": age_days,
|
|
}
|
|
|
|
# Ratio-based metrics (price vs reference)
|
|
for metric_key, cfg in RATIO_SCORERS.items():
|
|
price_val, price_date, price_age = _metric_observation(
|
|
index.get(cfg["price_key"], {}), date, cfg["price_key"]
|
|
)
|
|
# Try alternate price keys, each with an explicit freshness rule.
|
|
if price_val is None:
|
|
for pk in ["btc_price_coingecko", "btc_price_sma", "btc_price_lth"]:
|
|
price_val, price_date, price_age = _metric_observation(index.get(pk, {}), date, pk)
|
|
if price_val is not None:
|
|
break
|
|
ref_val, ref_date, ref_age = _metric_observation(
|
|
index.get(cfg["ref_key"], {}), date, cfg["ref_key"]
|
|
)
|
|
if price_val is not None and ref_val is not None and ref_val > 0:
|
|
pct_above = ((price_val - ref_val) / ref_val) * 100
|
|
s = _score_range(pct_above, cfg["ranges"])
|
|
if s is not None:
|
|
scores.append(s)
|
|
details[metric_key] = {
|
|
"value": pct_above,
|
|
"score": s,
|
|
"raw": pct_above,
|
|
"observed_date": min(price_date, ref_date),
|
|
"age_days": max(price_age, ref_age),
|
|
}
|
|
|
|
# Drawdown
|
|
dd = drawdowns.get(date)
|
|
if dd is not None:
|
|
s = _score_range(dd, DRAWDOWN_RANGES)
|
|
if s is not None:
|
|
scores.append(s)
|
|
details["drawdown"] = {"value": dd, "score": s, "raw": dd}
|
|
|
|
if not scores:
|
|
return None, details, 0
|
|
|
|
if ml_weights:
|
|
# ML-weighted composite
|
|
weighted_sum = 0.0
|
|
weight_total = 0.0
|
|
for metric_key, info in details.items():
|
|
ml_key = _BT_ML_KEY_MAP.get(metric_key, metric_key)
|
|
w = ml_weights.get(ml_key, 0.0)
|
|
weighted_sum += info["score"] * w
|
|
weight_total += w
|
|
if weight_total > 0:
|
|
composite = weighted_sum / weight_total * 10
|
|
else:
|
|
composite = sum(scores) / len(scores) * 10
|
|
else:
|
|
composite = sum(scores) / len(scores) * 10
|
|
return round(composite, 1), details, len(scores)
|
|
|
|
|
|
def compute_forward_returns(price_lookup, dates_sorted):
|
|
"""Precompute forward returns for all dates."""
|
|
periods = [30, 90, 180, 365]
|
|
returns = {}
|
|
for d in dates_sorted:
|
|
p0 = price_lookup.get(d)
|
|
if p0 is None or p0 <= 0:
|
|
continue
|
|
r = {}
|
|
dt = datetime.strptime(d, "%Y-%m-%d")
|
|
for days in periods:
|
|
future = (dt + timedelta(days=days)).strftime("%Y-%m-%d")
|
|
pf = price_lookup.get(future)
|
|
if pf is not None:
|
|
r[f"{days}d"] = round(((pf - p0) / p0) * 100, 2)
|
|
if r:
|
|
returns[d] = r
|
|
return returns
|
|
|
|
|
|
def compute_max_drawdown_forward(price_lookup, date, window=90):
|
|
"""Compute max drawdown within N days after a given date."""
|
|
dt = datetime.strptime(date, "%Y-%m-%d")
|
|
p0 = price_lookup.get(date)
|
|
if p0 is None or p0 <= 0:
|
|
return None
|
|
peak = p0
|
|
max_dd = 0
|
|
for i in range(1, window + 1):
|
|
future = (dt + timedelta(days=i)).strftime("%Y-%m-%d")
|
|
pf = price_lookup.get(future)
|
|
if pf is None:
|
|
continue
|
|
if pf > peak:
|
|
peak = pf
|
|
dd = ((peak - pf) / peak) * 100
|
|
if dd > max_dd:
|
|
max_dd = dd
|
|
return round(max_dd, 2) if max_dd > 0 else 0
|
|
|
|
|
|
def _file_signature(path):
|
|
"""Return a cheap signature that invalidates when an input file changes."""
|
|
try:
|
|
stat = os.stat(path)
|
|
return path, stat.st_mtime_ns, stat.st_size
|
|
except OSError:
|
|
return path, None, None
|
|
|
|
|
|
def clear_backtest_cache():
|
|
"""Clear memoized backtest results (primarily for explicit refreshes/tests)."""
|
|
with _BACKTEST_CACHE_LOCK:
|
|
_BACKTEST_CACHE.clear()
|
|
|
|
|
|
def _add_return_statistics(stats, period, returns):
|
|
"""Add return summaries and a moving-block-bootstrap mean interval."""
|
|
horizon_days = int(period.removesuffix("d"))
|
|
summary = summarize_returns(
|
|
returns,
|
|
block_size=min(horizon_days, len(returns)),
|
|
n_resamples=400,
|
|
)
|
|
stats[f"avg_{period}"] = summary["mean"]
|
|
stats[f"median_{period}"] = summary["median"]
|
|
stats[f"win_rate_{period}"] = summary["win_rate"]
|
|
stats[f"avg_{period}_ci_low"] = summary["mean_ci_low"]
|
|
stats[f"avg_{period}_ci_high"] = summary["mean_ci_high"]
|
|
stats[f"max_gain_{period}"] = round(max(returns), 2)
|
|
stats[f"max_loss_{period}"] = round(min(returns), 2)
|
|
stats[f"n_{period}"] = summary["n"]
|
|
|
|
|
|
def run_backtest(ml_mode=False):
|
|
"""Return an isolated cached result keyed by all material input files."""
|
|
signature = (
|
|
bool(ml_mode),
|
|
_file_signature(HISTORY_PATH),
|
|
_file_signature(_THRESH_PATH),
|
|
_file_signature(ML_WEIGHTS_PATH),
|
|
_file_signature(CACHE_PATH),
|
|
)
|
|
with _BACKTEST_CACHE_LOCK:
|
|
cached = _BACKTEST_CACHE.get(signature)
|
|
if cached is not None:
|
|
return copy.deepcopy(cached)
|
|
|
|
result = _compute_backtest(ml_mode=ml_mode)
|
|
with _BACKTEST_CACHE_LOCK:
|
|
_BACKTEST_CACHE[signature] = copy.deepcopy(result)
|
|
while len(_BACKTEST_CACHE) > _BACKTEST_CACHE_LIMIT:
|
|
_BACKTEST_CACHE.pop(next(iter(_BACKTEST_CACHE)))
|
|
return copy.deepcopy(result)
|
|
|
|
|
|
def _compute_backtest(ml_mode=False):
|
|
"""Run the full backtest and return comprehensive results.
|
|
|
|
If ml_mode=True, uses ML-optimized metric weights instead of equal weights.
|
|
"""
|
|
log.info("Loading historical data... (ml_mode=%s)", ml_mode)
|
|
if not os.path.exists(HISTORY_PATH):
|
|
return {"error": "No historical data found. Run history collector first."}
|
|
|
|
with open(HISTORY_PATH) as f:
|
|
history = json.load(f)
|
|
|
|
index = _build_daily_index(history)
|
|
|
|
# Build price lookup (prefer coingecko for completeness)
|
|
price_lookup = {}
|
|
for pk in ["btc_price_coingecko", "btc_price", "btc_price_sma", "btc_price_lth"]:
|
|
if pk in index:
|
|
for d, v in index[pk].items():
|
|
if d not in price_lookup:
|
|
price_lookup[d] = v
|
|
|
|
all_dates = _get_all_dates(index)
|
|
if not all_dates:
|
|
return {"error": "No date data available."}
|
|
|
|
log.info("Date range: %s to %s (%d days)", all_dates[0], all_dates[-1], len(all_dates))
|
|
|
|
# Compute drawdowns
|
|
drawdowns = _compute_ath_series(price_lookup, all_dates)
|
|
|
|
# Precompute forward returns
|
|
log.info("Computing forward returns...")
|
|
fwd_returns = compute_forward_returns(price_lookup, all_dates)
|
|
|
|
# Build an explicit evaluation plan. Fold-specific validation weights are OOS;
|
|
# final weights fitted on full history are never represented as OOS.
|
|
ml_plan = None
|
|
ml_artifact = None
|
|
ml_artifact_status = None
|
|
if ml_mode:
|
|
ml_artifact, ml_artifact_status = _load_ml_artifact()
|
|
if ml_artifact:
|
|
ml_plan = _build_ml_backtest_plan(ml_artifact)
|
|
else:
|
|
log.warning("ML mode requested with invalid artifact — falling back to equal weights")
|
|
|
|
# Score each day
|
|
log.info("Scoring %d days...", len(all_dates))
|
|
daily_scores = []
|
|
for d in all_dates:
|
|
ml_weights = None
|
|
ml_fold = None
|
|
if ml_plan:
|
|
ml_weights, ml_fold = _weights_for_backtest_date(d, ml_plan)
|
|
if ml_plan["is_out_of_sample"] and ml_weights is None:
|
|
continue
|
|
composite, details, n_metrics = score_day(d, index, drawdowns, ml_weights=ml_weights)
|
|
if composite is not None and n_metrics >= 3: # Require at least 3 metrics
|
|
price = price_lookup.get(d)
|
|
# Collect raw metric values for per-metric historical exploration
|
|
metric_values = {}
|
|
for mk, info in details.items():
|
|
raw = info.get("raw")
|
|
if raw is not None:
|
|
metric_values[mk] = round(raw, 6) if isinstance(raw, float) else raw
|
|
entry = {
|
|
"date": d,
|
|
"score": composite,
|
|
"n_metrics": n_metrics,
|
|
"price": price,
|
|
"forward_returns": fwd_returns.get(d, {}),
|
|
"metric_values": metric_values,
|
|
}
|
|
if ml_fold is not None:
|
|
entry["ml_fold"] = ml_fold
|
|
daily_scores.append(entry)
|
|
|
|
if not daily_scores:
|
|
return {"error": "No scored days (insufficient metric overlap)."}
|
|
|
|
log.info("Scored %d days with 3+ metrics", len(daily_scores))
|
|
|
|
# --- Bracket statistics ---
|
|
bracket_stats = []
|
|
for low, high, label in BRACKETS:
|
|
days_in = [d for d in daily_scores if score_in_bracket(d["score"], (low, high, label))]
|
|
if not days_in:
|
|
bracket_stats.append({
|
|
"range": f"{low}-{high}", "label": label, "days": 0,
|
|
})
|
|
continue
|
|
|
|
stats = {"range": f"{low}-{high}", "label": label, "days": len(days_in)}
|
|
for period in ["30d", "90d", "180d", "365d"]:
|
|
returns = [d["forward_returns"][period] for d in days_in if period in d["forward_returns"]]
|
|
if returns:
|
|
_add_return_statistics(stats, period, returns)
|
|
|
|
# Average max drawdown within 90 days
|
|
dd_list = []
|
|
for d in days_in:
|
|
dd = compute_max_drawdown_forward(price_lookup, d["date"], 90)
|
|
if dd is not None:
|
|
dd_list.append(dd)
|
|
if dd_list:
|
|
stats["avg_max_drawdown_90d"] = round(sum(dd_list) / len(dd_list), 2)
|
|
|
|
bracket_stats.append(stats)
|
|
|
|
# --- Peak signal events ---
|
|
signal_events = []
|
|
thresholds = [90, 80, 70]
|
|
for thresh in thresholds:
|
|
prev_score = 0
|
|
for d in daily_scores:
|
|
if d["score"] >= thresh and prev_score < thresh:
|
|
event = {
|
|
"date": d["date"],
|
|
"score": d["score"],
|
|
"threshold": thresh,
|
|
"price": d["price"],
|
|
"forward_returns": d["forward_returns"],
|
|
}
|
|
# Add future prices
|
|
if d["price"]:
|
|
dt = datetime.strptime(d["date"], "%Y-%m-%d")
|
|
for days_ahead in [30, 90, 365]:
|
|
future = (dt + timedelta(days=days_ahead)).strftime("%Y-%m-%d")
|
|
fp = price_lookup.get(future)
|
|
if fp:
|
|
event[f"price_{days_ahead}d"] = round(fp, 2)
|
|
signal_events.append(event)
|
|
prev_score = d["score"]
|
|
|
|
signal_events.sort(key=lambda e: e["date"])
|
|
|
|
# --- Current signal context ---
|
|
all_scores_list = [d["score"] for d in daily_scores]
|
|
all_scores_list.sort()
|
|
|
|
# Get current score from cache, recomputed on the common historical panel.
|
|
current_score = None
|
|
current_price = None
|
|
current_coverage = None
|
|
if os.path.exists(CACHE_PATH):
|
|
try:
|
|
with open(CACHE_PATH) as f:
|
|
cache = json.load(f)
|
|
scored = cache.get("_scored", {})
|
|
current_ml_weights = ml_artifact.get("weights") if ml_mode and ml_artifact else None
|
|
current_score, current_coverage = _common_panel_current_score(scored, current_ml_weights)
|
|
current_price = cache.get("price", {}).get("price")
|
|
except Exception:
|
|
pass
|
|
|
|
# If no comparable cache panel is available, use latest historical score.
|
|
if current_score is None and daily_scores:
|
|
current_score = daily_scores[-1]["score"]
|
|
current_price = daily_scores[-1].get("price")
|
|
current_coverage = {
|
|
"available_count": daily_scores[-1]["n_metrics"],
|
|
"panel_count": len(BACKTEST_METRIC_PANEL),
|
|
"available_keys": list(daily_scores[-1].get("metric_values", {})),
|
|
}
|
|
|
|
current_context = None
|
|
if current_score is not None:
|
|
# Percentile
|
|
below = len([s for s in all_scores_list if s <= current_score])
|
|
percentile = round(below / len(all_scores_list) * 100, 1)
|
|
|
|
# Find comparable historical periods
|
|
comparable = []
|
|
margin = 5
|
|
for d in daily_scores:
|
|
if abs(d["score"] - current_score) <= margin and d["forward_returns"]:
|
|
comparable.append(d)
|
|
|
|
avg_returns = {}
|
|
if comparable:
|
|
for period in ["30d", "90d", "180d", "365d"]:
|
|
vals = [d["forward_returns"][period] for d in comparable if period in d["forward_returns"]]
|
|
if vals:
|
|
avg_returns[period] = round(sum(vals) / len(vals), 2)
|
|
avg_1yr = avg_returns.get("365d")
|
|
|
|
# Best comparable examples — one per market cycle for diversity
|
|
# Cycles: pre-2016, 2016-2017 bull, 2018-2019 bear, 2020-2021 bull, 2022-2023 bear, 2024+
|
|
cycle_bins = [
|
|
("pre-2016", "2010-01-01", "2015-12-31"),
|
|
("2016-17 Bull", "2016-01-01", "2017-12-31"),
|
|
("2018-19 Bear", "2018-01-01", "2019-12-31"),
|
|
("2020-21 Bull", "2020-01-01", "2021-12-31"),
|
|
("2022-23 Bear", "2022-01-01", "2023-12-31"),
|
|
("2024+", "2024-01-01", "2099-12-31"),
|
|
]
|
|
examples = []
|
|
used_cycles = set()
|
|
# Sort comparable by closest score first, then pick one per cycle
|
|
sorted_comp = sorted(comparable, key=lambda d: abs(d["score"] - current_score))
|
|
for d in sorted_comp:
|
|
cycle_label = None
|
|
for label, start, end in cycle_bins:
|
|
if start <= d["date"] <= end:
|
|
cycle_label = label
|
|
break
|
|
if cycle_label and cycle_label not in used_cycles:
|
|
used_cycles.add(cycle_label)
|
|
examples.append({
|
|
"date": d["date"],
|
|
"score": d["score"],
|
|
"price": d["price"],
|
|
"forward_returns": d["forward_returns"],
|
|
"cycle": cycle_label,
|
|
})
|
|
if len(examples) >= 6:
|
|
break
|
|
# Sort examples chronologically
|
|
examples.sort(key=lambda d: d["date"])
|
|
|
|
current_context = {
|
|
"current_score": current_score,
|
|
"current_price": current_price,
|
|
"score_version": SCORE_VERSION,
|
|
"metric_panel_id": "historical-common-v1",
|
|
"coverage": current_coverage,
|
|
"current_weighting_source": (
|
|
"final_full_history_weights" if ml_mode and ml_artifact else "equal_weight"
|
|
),
|
|
"percentile": percentile,
|
|
"comparable_days": len(comparable),
|
|
"avg_1yr_return": avg_1yr,
|
|
"avg_30d_return": avg_returns.get("30d"),
|
|
"avg_90d_return": avg_returns.get("90d"),
|
|
"avg_180d_return": avg_returns.get("180d"),
|
|
"examples": examples,
|
|
}
|
|
|
|
# --- Build time series for charting ---
|
|
# Smart downsampling: daily for last 2 years, weekly before that
|
|
# Include per-metric values so the frontend can plot any metric.
|
|
chart_data = []
|
|
import datetime as _dt
|
|
try:
|
|
last_date = _dt.datetime.strptime(daily_scores[-1]["date"], "%Y-%m-%d")
|
|
cutoff_date = (last_date - _dt.timedelta(days=730)).strftime("%Y-%m-%d")
|
|
except Exception:
|
|
cutoff_date = "2024-01-01"
|
|
|
|
# Collect all metric keys that were ever scored (for per-metric series)
|
|
all_metric_keys = set()
|
|
for d in daily_scores:
|
|
all_metric_keys.update(d.get("metric_values", {}).keys())
|
|
|
|
for i, d in enumerate(daily_scores):
|
|
is_recent = d["date"] >= cutoff_date
|
|
if is_recent or i % 7 == 0 or i == len(daily_scores) - 1:
|
|
entry = {
|
|
"date": d["date"],
|
|
"score": d["score"],
|
|
"price": d["price"],
|
|
}
|
|
# Include per-metric values (raw metric value, not score)
|
|
metric_vals = d.get("metric_values", {})
|
|
if metric_vals:
|
|
entry["metric_values"] = metric_vals
|
|
chart_data.append(entry)
|
|
|
|
if not ml_mode:
|
|
ml_evaluation = {"requested": False, "is_out_of_sample": False}
|
|
elif ml_plan:
|
|
ml_evaluation = {
|
|
"requested": True,
|
|
"evaluation_scope": ml_plan["evaluation_scope"],
|
|
"is_out_of_sample": ml_plan["is_out_of_sample"],
|
|
"weighting_source": ml_plan["weighting_source"],
|
|
"folds": len(ml_plan["folds"]),
|
|
"artifact": ml_artifact_status,
|
|
}
|
|
else:
|
|
ml_evaluation = {
|
|
"requested": True,
|
|
"evaluation_scope": "equal_weight_fallback",
|
|
"is_out_of_sample": False,
|
|
"weighting_source": "none_invalid_artifact",
|
|
"folds": 0,
|
|
"artifact": ml_artifact_status,
|
|
}
|
|
|
|
data_quality = _backtest_data_quality_metadata([day["n_metrics"] for day in daily_scores])
|
|
result = {
|
|
"date_range": {"start": daily_scores[0]["date"], "end": daily_scores[-1]["date"]},
|
|
"total_days_scored": len(daily_scores),
|
|
"metric_panel": data_quality["metric_panel"],
|
|
"coverage": data_quality["coverage"],
|
|
"staleness_days": data_quality["staleness_days"],
|
|
"bracket_stats": bracket_stats,
|
|
"signal_events": signal_events,
|
|
"current_context": current_context,
|
|
"chart_data": chart_data,
|
|
"ml_mode": ml_mode,
|
|
"ml_evaluation": ml_evaluation,
|
|
"score_version": SCORE_VERSION,
|
|
"computed_at": datetime.utcnow().isoformat() + "Z",
|
|
}
|
|
|
|
log.info("Backtest complete: %d days, %d signal events", len(daily_scores), len(signal_events))
|
|
return result
|