From 661579abf9b3b1f1634f856bc8db21be0f7778f9 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Sun, 26 Jul 2026 23:06:59 +0000 Subject: [PATCH] fix: publish historical metric coverage --- backtesting/engine.py | 139 +++++++++++++++++++++++++--- scoring/engine.py | 12 +++ tests/test_backtest_data_quality.py | 53 +++++++++++ 3 files changed, 192 insertions(+), 12 deletions(-) create mode 100644 tests/test_backtest_data_quality.py diff --git a/backtesting/engine.py b/backtesting/engine.py index a584d82..7cbff96 100644 --- a/backtesting/engine.py +++ b/backtesting/engine.py @@ -54,6 +54,22 @@ RATIO_SCORERS = { }, } +BACKTEST_METRIC_PANEL = tuple(METRIC_SCORERS) + tuple(RATIO_SCORERS) + ("drawdown",) +METRIC_MAX_AGE_DAYS = { + "fear_greed": 2, + "puell_multiple": 7, + "mvrv_zscore": 7, + "reserve_risk": 7, + "rhodl_ratio": 7, + "nupl": 7, + "btc_price": 3, + "btc_price_coingecko": 3, + "btc_price_sma": 3, + "btc_price_lth": 3, + "200w_sma": 7, + "lth_realized_price": 7, +} + DRAWDOWN_RANGES = _THRESH.get("drawdown", {}).get("ranges", [[60, None, 10], [40, 60, 8], [25, 40, 6], [15, 25, 4], [5, 15, 2], [None, 5, 0]]) @@ -90,8 +106,8 @@ def _get_all_dates(index): return sorted(all_dates) -def _last_known_value(lookup, date, max_lookback=30): - """Get value for date, or most recent prior value within lookback window.""" +def _last_known_value(lookup, date, max_lookback=0): + """Get value for date, or a prior value within an explicit lookback.""" if date in lookup: return lookup[date] d = datetime.strptime(date, "%Y-%m-%d") @@ -102,6 +118,17 @@ def _last_known_value(lookup, date, max_lookback=30): return None +def _metric_observation(lookup, date, metric_key): + """Return value, source date, and age under a metric-specific freshness rule.""" + max_age = METRIC_MAX_AGE_DAYS.get(metric_key, 0) + target = datetime.strptime(date, "%Y-%m-%d") + for age in range(max_age + 1): + source_date = (target - timedelta(days=age)).strftime("%Y-%m-%d") + if source_date in lookup: + return lookup[source_date], source_date, age + return None, None, None + + def _compute_ath_series(price_lookup, dates): """Compute running ATH and drawdown for each date.""" ath = 0 @@ -196,6 +223,58 @@ _BT_ML_KEY_MAP = { } +def _common_panel_current_score(scored, ml_weights=None): + """Recompute the current score using only metrics present historically.""" + by_key = { + metric.get("key"): metric.get("score") + for metric in scored.get("metrics", []) + if metric.get("key") in BACKTEST_METRIC_PANEL and metric.get("score") is not None + } + available_keys = [key for key in BACKTEST_METRIC_PANEL if key in by_key] + coverage = { + "available_count": len(available_keys), + "panel_count": len(BACKTEST_METRIC_PANEL), + "available_keys": available_keys, + } + if not available_keys: + return None, coverage + + if ml_weights: + weighted = [ + (by_key[key], ml_weights.get(_BT_ML_KEY_MAP[key], 0.0)) + for key in available_keys + ] + weight_total = sum(weight for _, weight in weighted) + if weight_total > 0: + return round(sum(score * weight for score, weight in weighted) / weight_total * 10, 1), coverage + + return round(sum(by_key[key] for key in available_keys) / len(available_keys) * 10, 1), coverage + + +def _backtest_data_quality_metadata(metric_counts): + """Describe historical panel, coverage, and freshness assumptions.""" + coverage = { + "minimum_metrics": min(metric_counts), + "maximum_metrics": max(metric_counts), + "average_metrics": round(sum(metric_counts) / len(metric_counts), 1), + "panel_count": len(BACKTEST_METRIC_PANEL), + } if metric_counts else { + "minimum_metrics": 0, + "maximum_metrics": 0, + "average_metrics": 0, + "panel_count": len(BACKTEST_METRIC_PANEL), + } + return { + "metric_panel": { + "id": "historical-common-v1", + "keys": list(BACKTEST_METRIC_PANEL), + "count": len(BACKTEST_METRIC_PANEL), + }, + "coverage": coverage, + "staleness_days": dict(METRIC_MAX_AGE_DAYS), + } + + def score_day(date, index, drawdowns, ml_weights=None): """Score a single day using all available metrics. Returns (composite_score, details, n_metrics). @@ -207,29 +286,47 @@ def score_day(date, index, drawdowns, ml_weights=None): # Simple range-based metrics for metric_key, cfg in METRIC_SCORERS.items(): - val = _last_known_value(index.get(metric_key, {}), date) + val, observed_date, age_days = _metric_observation( + index.get(metric_key, {}), date, metric_key + ) if val is not None: s = _score_range(val, cfg["ranges"]) if s is not None: scores.append(s) - details[metric_key] = {"value": val, "score": s, "raw": val} + details[metric_key] = { + "value": val, + "score": s, + "raw": val, + "observed_date": observed_date, + "age_days": age_days, + } # Ratio-based metrics (price vs reference) for metric_key, cfg in RATIO_SCORERS.items(): - price_val = _last_known_value(index.get(cfg["price_key"], {}), date) - # Try alternate price keys + price_val, price_date, price_age = _metric_observation( + index.get(cfg["price_key"], {}), date, cfg["price_key"] + ) + # Try alternate price keys, each with an explicit freshness rule. if price_val is None: for pk in ["btc_price_coingecko", "btc_price_sma", "btc_price_lth"]: - price_val = _last_known_value(index.get(pk, {}), date) + price_val, price_date, price_age = _metric_observation(index.get(pk, {}), date, pk) if price_val is not None: break - ref_val = _last_known_value(index.get(cfg["ref_key"], {}), date) + ref_val, ref_date, ref_age = _metric_observation( + index.get(cfg["ref_key"], {}), date, cfg["ref_key"] + ) if price_val is not None and ref_val is not None and ref_val > 0: pct_above = ((price_val - ref_val) / ref_val) * 100 s = _score_range(pct_above, cfg["ranges"]) if s is not None: scores.append(s) - details[metric_key] = {"value": pct_above, "score": s, "raw": pct_above} + details[metric_key] = { + "value": pct_above, + "score": s, + "raw": pct_above, + "observed_date": min(price_date, ref_date), + "age_days": max(price_age, ref_age), + } # Drawdown dd = drawdowns.get(date) @@ -339,6 +436,7 @@ def run_backtest(ml_mode=False): # Build an explicit evaluation plan. Fold-specific validation weights are OOS; # final weights fitted on full history are never represented as OOS. ml_plan = None + ml_artifact = None ml_artifact_status = None if ml_mode: ml_artifact, ml_artifact_status = _load_ml_artifact() @@ -447,23 +545,30 @@ def run_backtest(ml_mode=False): all_scores_list = [d["score"] for d in daily_scores] all_scores_list.sort() - # Get current score from cache + # Get current score from cache, recomputed on the common historical panel. current_score = None current_price = None + current_coverage = None if os.path.exists(CACHE_PATH): try: with open(CACHE_PATH) as f: cache = json.load(f) scored = cache.get("_scored", {}) - current_score = scored.get("composite_score") + current_ml_weights = ml_artifact.get("weights") if ml_mode and ml_artifact else None + current_score, current_coverage = _common_panel_current_score(scored, current_ml_weights) current_price = cache.get("price", {}).get("price") except Exception: pass - # If no cache, use latest daily score + # If no comparable cache panel is available, use latest historical score. if current_score is None and daily_scores: current_score = daily_scores[-1]["score"] current_price = daily_scores[-1].get("price") + current_coverage = { + "available_count": daily_scores[-1]["n_metrics"], + "panel_count": len(BACKTEST_METRIC_PANEL), + "available_keys": list(daily_scores[-1].get("metric_values", {})), + } current_context = None if current_score is not None: @@ -523,6 +628,12 @@ def run_backtest(ml_mode=False): current_context = { "current_score": current_score, "current_price": current_price, + "score_version": SCORE_VERSION, + "metric_panel_id": "historical-common-v1", + "coverage": current_coverage, + "current_weighting_source": ( + "final_full_history_weights" if ml_mode and ml_artifact else "equal_weight" + ), "percentile": percentile, "comparable_days": len(comparable), "avg_1yr_return": avg_1yr, @@ -583,9 +694,13 @@ def run_backtest(ml_mode=False): "artifact": ml_artifact_status, } + data_quality = _backtest_data_quality_metadata([day["n_metrics"] for day in daily_scores]) result = { "date_range": {"start": daily_scores[0]["date"], "end": daily_scores[-1]["date"]}, "total_days_scored": len(daily_scores), + "metric_panel": data_quality["metric_panel"], + "coverage": data_quality["coverage"], + "staleness_days": data_quality["staleness_days"], "bracket_stats": bracket_stats, "signal_events": signal_events, "current_context": current_context, diff --git a/scoring/engine.py b/scoring/engine.py index 1f00c22..74aa901 100644 --- a/scoring/engine.py +++ b/scoring/engine.py @@ -556,6 +556,16 @@ def score_all(metrics): "scored_count": len(valid_scores), "total_count": len(results), "score_version": SCORE_VERSION, + "metric_panel": { + "id": "live-all-v1", + "keys": [result["key"] for result in results], + "count": len(results), + }, + "coverage": { + "available_count": len(valid_scores), + "panel_count": len(results), + "ratio": len(valid_scores) / len(results), + }, } @@ -669,4 +679,6 @@ def score_all_ml(metrics): "classic_score": classic["composite_score"], "ml_weight_total": round(weight_total, 4), "score_version": SCORE_VERSION, + "metric_panel": classic["metric_panel"], + "coverage": classic["coverage"], } diff --git a/tests/test_backtest_data_quality.py b/tests/test_backtest_data_quality.py new file mode 100644 index 0000000..0383e49 --- /dev/null +++ b/tests/test_backtest_data_quality.py @@ -0,0 +1,53 @@ +from backtesting import engine as backtest +from scoring import engine as scoring + + +def test_metric_specific_staleness_does_not_apply_generic_30_day_fill(): + lookup = {"2024-01-01": 42} + + assert backtest._metric_observation(lookup, "2024-01-03", "fear_greed") == (42, "2024-01-01", 2) + assert backtest._metric_observation(lookup, "2024-01-04", "fear_greed") == (None, None, None) + assert backtest._metric_observation(lookup, "2024-01-08", "200w_sma") == (42, "2024-01-01", 7) + assert backtest._metric_observation(lookup, "2024-01-02", "unknown_metric") == (None, None, None) + + +def test_current_context_score_uses_only_the_common_backtest_panel(): + cached_scored = { + "composite_score": 100, + "metrics": [ + {"key": "fear_greed", "score": 10}, + {"key": "puell_multiple", "score": 0}, + {"key": "sopr", "score": 10}, + {"key": "vdd_multiple", "score": 10}, + ], + } + + score, coverage = backtest._common_panel_current_score(cached_scored) + + assert score == 50.0 + assert coverage == { + "available_count": 2, + "panel_count": len(backtest.BACKTEST_METRIC_PANEL), + "available_keys": ["fear_greed", "puell_multiple"], + } + + +def test_live_and_backtest_outputs_publish_panel_and_coverage_metadata(): + live = scoring.score_all({"fear_greed": {"value": 10}}) + + assert live["metric_panel"]["id"] == "live-all-v1" + assert live["metric_panel"]["count"] == live["total_count"] + assert live["coverage"]["available_count"] == live["scored_count"] + assert live["coverage"]["ratio"] == live["scored_count"] / live["total_count"] + assert "score_version" in live + + metadata = backtest._backtest_data_quality_metadata([3, 5, 9]) + assert metadata["metric_panel"]["id"] == "historical-common-v1" + assert metadata["metric_panel"]["keys"] == list(backtest.BACKTEST_METRIC_PANEL) + assert metadata["coverage"] == { + "minimum_metrics": 3, + "maximum_metrics": 9, + "average_metrics": 5.7, + "panel_count": 9, + } + assert metadata["staleness_days"] == backtest.METRIC_MAX_AGE_DAYS