diff --git a/backtesting/engine.py b/backtesting/engine.py index 1a161a4..1ce03fb 100644 --- a/backtesting/engine.py +++ b/backtesting/engine.py @@ -146,9 +146,10 @@ _BT_ML_KEY_MAP = { def score_day(date, index, drawdowns, ml_weights=None): - """Score a single day using all available metrics. Returns (composite_score, individual_scores, n_metrics). + """Score a single day using all available metrics. Returns (composite_score, details, n_metrics). If ml_weights is provided, uses ML-optimized weighting instead of equal weights. + details includes both "score" and "raw" (the actual metric value before scoring). """ scores = [] details = {} @@ -160,7 +161,7 @@ def score_day(date, index, drawdowns, ml_weights=None): s = _score_range(val, cfg["ranges"]) if s is not None: scores.append(s) - details[metric_key] = {"value": val, "score": s} + details[metric_key] = {"value": val, "score": s, "raw": val} # Ratio-based metrics (price vs reference) for metric_key, cfg in RATIO_SCORERS.items(): @@ -177,7 +178,7 @@ def score_day(date, index, drawdowns, ml_weights=None): s = _score_range(pct_above, cfg["ranges"]) if s is not None: scores.append(s) - details[metric_key] = {"value": pct_above, "score": s} + details[metric_key] = {"value": pct_above, "score": s, "raw": pct_above} # Drawdown dd = drawdowns.get(date) @@ -185,7 +186,7 @@ def score_day(date, index, drawdowns, ml_weights=None): s = _score_range(dd, DRAWDOWN_RANGES) if s is not None: scores.append(s) - details["drawdown"] = {"value": dd, "score": s} + details["drawdown"] = {"value": dd, "score": s, "raw": dd} if not scores: return None, details, 0 @@ -297,12 +298,19 @@ def run_backtest(ml_mode=False): composite, details, n_metrics = score_day(d, index, drawdowns, ml_weights=ml_weights) if composite is not None and n_metrics >= 3: # Require at least 3 metrics price = price_lookup.get(d) + # Collect raw metric values for per-metric historical exploration + metric_values = {} + for mk, info in details.items(): + raw = info.get("raw") + if raw is not None: + metric_values[mk] = round(raw, 6) if isinstance(raw, float) else raw entry = { "date": d, "score": composite, "n_metrics": n_metrics, "price": price, "forward_returns": fwd_returns.get(d, {}), + "metric_values": metric_values, } daily_scores.append(entry) @@ -462,6 +470,7 @@ def run_backtest(ml_mode=False): # --- Build time series for charting --- # Smart downsampling: daily for last 2 years, weekly before that + # Include per-metric values so the frontend can plot any metric. chart_data = [] import datetime as _dt try: @@ -469,14 +478,25 @@ def run_backtest(ml_mode=False): cutoff_date = (last_date - _dt.timedelta(days=730)).strftime("%Y-%m-%d") except Exception: cutoff_date = "2024-01-01" + + # Collect all metric keys that were ever scored (for per-metric series) + all_metric_keys = set() + for d in daily_scores: + all_metric_keys.update(d.get("metric_values", {}).keys()) + for i, d in enumerate(daily_scores): is_recent = d["date"] >= cutoff_date if is_recent or i % 7 == 0 or i == len(daily_scores) - 1: - chart_data.append({ + entry = { "date": d["date"], "score": d["score"], "price": d["price"], - }) + } + # Include per-metric values (raw metric value, not score) + metric_vals = d.get("metric_values", {}) + if metric_vals: + entry["metrics"] = metric_vals + chart_data.append(entry) result = { "date_range": {"start": daily_scores[0]["date"], "end": daily_scores[-1]["date"]}, diff --git a/dashboard/server.py b/dashboard/server.py index a3d55bc..ffc076f 100644 --- a/dashboard/server.py +++ b/dashboard/server.py @@ -165,7 +165,9 @@ def run_scrape(force_full=False): # 3. On-chain metrics — use cached values (historical data is permanent) onchain_keys = ["puell_multiple", "mvrv_zscore", "reserve_risk", "rhodl_ratio", "nupl", "200w_sma", "lth_realized_price", "hash_ribbons", - "pi_cycle_bottom", "lth_supply"] + "pi_cycle_bottom", "lth_supply", "sopr", "sellside_risk", + "active_address_momentum", "txcount_momentum", "nvt_price", + "vdd_multiple"] has_cached_onchain = any(existing_cache.get(k, {}).get("value") is not None for k in onchain_keys) @@ -176,6 +178,12 @@ def run_scrape(force_full=False): from scrapers import lookintobitcoin onchain = lookintobitcoin.scrape_all() metrics.update(onchain) + try: + from scrapers import checkonchain + metrics.update(checkonchain.scrape_all()) + except Exception as e: + log.error("CheckOnChain scraping failed: %s\n%s", e, traceback.format_exc()) + _last_error = f"CheckOnChain scraping failed: {e}" metrics["_onchain_timestamp"] = datetime.now(timezone.utc).isoformat() except Exception as e: log.error("LookIntoBitcoin scraping failed: %s\n%s", e, traceback.format_exc()) @@ -341,6 +349,46 @@ def _fetch_models(provider, providers): # ── API Routes ──────────────────────────────────────────────────────────── +def _with_informational_onchain_metrics(scored, cache): + """Add non-scored on-chain data cards without changing composite scoring.""" + if not isinstance(scored, dict): + return scored + + enriched = dict(scored) + metrics = [dict(m) for m in scored.get("metrics", [])] + existing_keys = {m.get("key") for m in metrics} + + lth_supply = cache.get("lth_supply", {}) + lth_value = lth_supply.get("value") + if lth_value is not None and "lth_supply" not in existing_keys: + trend = lth_supply.get("trend") + trend_text = f" — {trend}" if trend else "" + metrics.append({ + "name": "Long-Term Holder Supply", + "key": "lth_supply", + "value": lth_value, + "display_value": f"{lth_value:,.0f} BTC", + "score": None, + "description": "Informational on-chain metric; not included in the composite score" + trend_text, + "recent": lth_supply.get("recent", []), + }) + + pi_cycle = cache.get("pi_cycle_bottom", {}) + pi_value = pi_cycle.get("value") + if pi_value is not None and "pi_cycle_bottom" not in existing_keys: + metrics.append({ + "name": "Pi Cycle Bottom", + "key": "pi_cycle_bottom", + "value": pi_value, + "display_value": f"{pi_value:,.2f}" if isinstance(pi_value, (int, float)) else str(pi_value), + "score": None, + "description": "Informational on-chain cycle metric; not included in the composite score", + "recent": pi_cycle.get("recent", []), + }) + + enriched["metrics"] = metrics + return enriched + @app.get("/api/data") def api_data(mode: str = "classic"): """Return current cached metrics + scores. @@ -351,6 +399,7 @@ def api_data(mode: str = "classic"): scored = cache.get("_scored_ml", cache.get("_scored", {})) else: scored = cache.get("_scored", {}) + scored = _with_informational_onchain_metrics(scored, cache) price_data = cache.get("price", {}) drawdown_data = cache.get("drawdown", {}) extras = cache.get("price_extras", {}) @@ -503,8 +552,17 @@ DASHBOARD_HTML = """ .meta-row{display:flex;gap:16px;flex-wrap:wrap;margin-top:8px;font-size:.8rem;color:var(--text-dim)} .meta-row span{display:flex;align-items:center;gap:4px} .metrics-grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(300px,1fr));gap:12px;margin-bottom:20px} -.metric-card{background:var(--card);border-radius:10px;padding:14px;border:1px solid var(--border);transition:border-color .15s} +.metric-card{background:var(--card);border-radius:10px;padding:14px;border:1px solid var(--border);transition:border-color .15s;cursor:pointer} .metric-card:hover{border-color:var(--text-dim)} +.metric-card.selected{border-color:#a78bfa;box-shadow:0 0 0 1px #a78bfa,0 0 12px rgba(167,139,250,0.15)} +.metric-click-hint{font-size:.6rem;margin-left:4px;opacity:0;transition:opacity .15s} +.metric-card:hover .metric-click-hint{opacity:.5} +.metric-card.selected .metric-click-hint{opacity:1} +.mc-examples-title{font-size:.75rem;color:#94a3b8;text-transform:uppercase;letter-spacing:.06em;margin-bottom:6px} +.mc-example{font-size:.8rem;font-family:var(--mono);padding:4px 0;border-bottom:1px solid rgba(255,255,255,0.03)} +.mc-ex-date{color:#e2e8f0} +.mc-ex-cycle{color:#a78bfa;font-size:.7rem} +.mc-ex-price{color:#94a3b8} .metric-header{display:flex;justify-content:space-between;align-items:flex-start;margin-bottom:8px} .metric-name{font-size:.85rem;font-weight:600} .metric-score{display:flex;align-items:center;gap:6px} @@ -588,6 +646,20 @@ DASHBOARD_HTML = """ View full backtest → + + +

On-Chain Metrics

@@ -690,6 +762,8 @@ function drawSparkline(canvasId, data, color) { ctx.stroke(); } +let selectedMetric = null; + function renderMetrics(metrics) { const grid = document.getElementById('metricsGrid'); if (!metrics || !metrics.length) { @@ -703,10 +777,11 @@ function renderMetrics(metrics) { const color = m.score != null ? scoreColor(m.score, 10) : '#64748b'; const fillPct = m.score != null ? (m.score / 10 * 100) : 0; const hasSparkline = m.recent && m.recent.length > 2; + const isSelected = selectedMetric === m.key ? ' selected' : ''; - html += '
'; + html += '
'; html += '
'; - html += '
' + m.name + '
'; + html += '
' + m.name + '👆
'; html += '
'; html += '
'; html += '
' + score + '
'; @@ -733,11 +808,112 @@ function renderMetrics(metrics) { } }); }); + + // Attach click handlers to metric cards + document.querySelectorAll('.metric-card').forEach(card => { + card.addEventListener('click', function() { + const key = this.getAttribute('data-key'); + const name = this.getAttribute('data-name'); + if (key) selectMetric(key, name); + }); + }); +} + +// Metric selection + context panel +function selectMetric(metricKey, metricName) { + if (selectedMetric === metricKey) { + // Deselect if clicking the same one + selectedMetric = null; + window._highlightMetric = null; + document.getElementById('metricContextPanel').style.display = 'none'; + const panel = document.getElementById('histContext'); + if (panel) panel.style.display = 'block'; + applyChartRange(currentRange); // Re-render chart without highlight + } else { + selectedMetric = metricKey; + loadMetricContext(metricKey, metricName); + } + poll(); // Re-render metric cards with highlight +} + +async function loadMetricContext(metricKey, metricName) { + try { + const r = await fetch('/api/metric-context?metric=' + encodeURIComponent(metricKey) + '&mode=' + currentMode); + const ctx = await r.json(); + if (ctx.error) { + showToast(ctx.error, 'error'); + return; + } + + // Show metric context panel, hide composite context + const panel = document.getElementById('histContext'); + if (panel) panel.style.display = 'none'; + + const mcp = document.getElementById('metricContextPanel'); + mcp.style.display = 'block'; + + document.getElementById('mcTitle').textContent = metricName; + document.getElementById('mcCurrent').textContent = 'Current: ' + (ctx.current_raw != null ? ctx.current_raw : 'N/A'); + document.getElementById('mcPercentile').textContent = 'Metric value in top ' + (100 - ctx.percentile).toFixed(1) + '% historically'; + document.getElementById('mcComparable').textContent = ctx.comparable_days + ' comparable days found'; + + const fmtR = (v) => v == null ? '--' : (v >= 0 ? '+' : '') + v.toFixed(1) + '%'; + const cR = v => v != null && v >= 0 ? '#22c55e' : '#ef4444'; + const periods = [['30d', ctx.avg_30d_return], ['90d', ctx.avg_90d_return], ['180d', ctx.avg_180d_return], ['1yr', ctx.avg_1yr_return]]; + let retHtml = ''; + for (const [label, val] of periods) { + if (val != null) retHtml += '' + label + ': ' + fmtR(val) + ' · '; + } + document.getElementById('mcReturns').innerHTML = retHtml ? 'Avg returns when ' + metricName + ' was similar: ' + retHtml : 'No forward return data available'; + + // Examples + const exEl = document.getElementById('mcExamples'); + if (ctx.examples && ctx.examples.length) { + let exHtml = '
Historical examples:
'; + ctx.examples.forEach(ex => { + const fwd30 = ex.forward_returns['30d']; + const fwd365 = ex.forward_returns['365d']; + exHtml += '
'; + exHtml += '' + ex.date + ' '; + exHtml += '' + ex.cycle + ' '; + exHtml += '$' + (ex.price ? ex.price.toLocaleString() : 'N/A') + ''; + if (fwd30 != null) exHtml += ' 30d: ' + fmtR(fwd30) + ''; + if (fwd365 != null) exHtml += ' 1yr: ' + fmtR(fwd365) + ''; + exHtml += '
'; + }); + exEl.innerHTML = exHtml; + exEl.style.display = 'block'; + } else { + exEl.innerHTML = ''; + exEl.style.display = 'none'; + } + + // Highlight matching periods on the chart + highlightMetricPeriods(metricKey, ctx.current_raw, ctx.margin); + } catch(e) { + console.error('Metric context load failed:', e); + } +} + +function highlightMetricPeriods(metricKey, currentRaw, margin) { + if (!fullDailyScores || !currentRaw || margin == null) return; + + // Build an array of {date, rawValue} for the selected metric + const metricSeries = fullDailyScores + .filter(d => d.metrics && d.metrics[metricKey] != null) + .map(d => ({ date: d.date, value: d.metrics[metricKey], isSimilar: Math.abs(d.metrics[metricKey] - currentRaw) <= margin })); + + // Store for use in chart rendering + window._highlightMetric = { key: metricKey, series: metricSeries, currentRaw, margin }; + + // Re-render chart with highlight + applyChartRange(currentRange); } let histChart = null; let fullDailyScores = null; let currentRange = 0; // 0 = ALL +let currentMode = 'classic'; function renderHistory(history) { // Legacy: still called by loadData but we'll use backtest data instead @@ -784,7 +960,61 @@ function renderHistoryFromData(history) { }); } - // Accumulation zone backgrounds + // If a metric is selected, add its overlay + highlight similar periods + const highlight = window._highlightMetric; + let metricColor = '#a78bfa'; + if (highlight && highlight.series && highlight.series.length) { + // Build a sparse array aligned to current chart labels + const metricByDate = {}; + highlight.series.forEach(s => { metricByDate[s.date] = s; }); + const metricData = labels.map(l => { + const entry = metricByDate[l]; + return entry ? entry.value : null; + }); + const hasMetricData = metricData.some(v => v != null); + + if (hasMetricData) { + datasets.push({ + label: 'Selected Metric', + data: metricData, + borderColor: metricColor, + borderWidth: 1.5, + borderDash: [2, 2], + fill: false, + tension: 0.2, + pointRadius: 0, + yAxisID: 'y2', + }); + } + + // Highlight similar periods with point dots on the score line + const similarIndices = []; + labels.forEach((l, i) => { + const entry = metricByDate[l]; + if (entry && entry.isSimilar) similarIndices.push(i); + }); + + if (similarIndices.length) { + const highlightData = labels.map((l, i) => + similarIndices.includes(i) ? scores[i] : null + ); + datasets.push({ + label: 'Similar Periods', + data: highlightData, + borderColor: 'rgba(167,139,250,0)', + backgroundColor: '#a78bfa', + pointRadius: 3, + pointHoverRadius: 5, + showLine: false, + yAxisID: 'y', + }); + } + } + + // Determine y2 scale for the metric overlay + const hasMetricDataset = datasets.some(d => d.yAxisID === 'y2'); + + // Accumulation zone backgrounds + metric highlight bands const zonePlugin = { id: 'zones', beforeDraw(chart) { @@ -812,9 +1042,55 @@ function renderHistoryFromData(history) { ctx.stroke(); ctx.setLineDash([]); }); + + // Draw vertical highlight bands for similar periods + if (highlight && highlight.series) { + const metricByDate = {}; + highlight.series.forEach(s => { metricByDate[s.date] = s; }); + const xScale = chart.scales.x; + labels.forEach((l, i) => { + const entry = metricByDate[l]; + if (entry && entry.isSimilar) { + const x = xScale.getPixelForValue(i); + ctx.fillStyle = 'rgba(167,139,250,0.08)'; + ctx.fillRect(x - 3, top, 6, bottom - top); + } + }); + } } }; + const scales = { + x: { + ticks: { color: '#64748b', maxTicksLimit: 12, font: { family: 'monospace', size: 10 } }, + grid: { color: 'rgba(255,255,255,0.03)' } + }, + y: { + min: 0, max: 100, + ticks: { color: '#22d3ee', font: { family: 'monospace', size: 10 } }, + grid: { color: 'rgba(255,255,255,0.03)' }, + title: { display: true, text: 'Score', color: '#22d3ee', font: { family: 'monospace', size: 11 } } + }, + y1: { + position: 'right', + ticks: { + color: '#f7931a', + font: { family: 'monospace', size: 10 }, + callback: v => '$' + (v >= 1000 ? (v/1000).toFixed(0) + 'k' : v) + }, + grid: { drawOnChartArea: false }, + title: { display: true, text: 'BTC Price', color: '#f7931a', font: { family: 'monospace', size: 11 } } + }, + }; + + if (hasMetricDataset) { + scales['y2'] = { + position: 'right', + display: false, + grid: { drawOnChartArea: false }, + }; + } + histChart = new Chart(ctx, { type: 'line', plugins: [zonePlugin], @@ -834,6 +1110,8 @@ function renderHistoryFromData(history) { callbacks: { label: function(ctx) { if (ctx.dataset.yAxisID === 'y1') return 'BTC: $' + ctx.raw.toLocaleString(); + if (ctx.dataset.yAxisID === 'y2') return 'Metric: ' + (ctx.raw != null ? ctx.raw.toFixed(4) : 'N/A'); + if (ctx.dataset.label === 'Similar Periods') return '★ Similar period (Score: ' + ctx.raw.toFixed(1) + ')'; const s = ctx.raw; let zone = s >= 80 ? 'Extreme Accum' : s >= 65 ? 'Strong Accum' : s >= 50 ? 'Moderate' : s >= 35 ? 'Neutral' : 'Caution'; return 'Score: ' + s.toFixed(1) + ' (' + zone + ')'; @@ -841,28 +1119,7 @@ function renderHistoryFromData(history) { } } }, - scales: { - x: { - ticks: { color: '#64748b', maxTicksLimit: 12, font: { family: 'monospace', size: 10 } }, - grid: { color: 'rgba(255,255,255,0.03)' } - }, - y: { - min: 0, max: 100, - ticks: { color: '#22d3ee', font: { family: 'monospace', size: 10 } }, - grid: { color: 'rgba(255,255,255,0.03)' }, - title: { display: true, text: 'Score', color: '#22d3ee', font: { family: 'monospace', size: 11 } } - }, - y1: { - position: 'right', - ticks: { - color: '#f7931a', - font: { family: 'monospace', size: 10 }, - callback: v => '$' + (v >= 1000 ? (v/1000).toFixed(0) + 'k' : v) - }, - grid: { drawOnChartArea: false }, - title: { display: true, text: 'BTC Price', color: '#f7931a', font: { family: 'monospace', size: 11 } } - } - } + scales, } }); } @@ -992,8 +1249,6 @@ async function doRefresh(full) { setTimeout(() => { btn.disabled = false; btn.textContent = origText; }, delay); } -let currentMode = 'classic'; - function setMode(mode) { currentMode = mode; document.querySelectorAll('.mode-btn').forEach(b => { @@ -1286,6 +1541,245 @@ def api_backtest_status(): return status +@app.get("/api/metric-context") +def api_metric_context(metric: str, margin: float = 0.0, mode: str = "classic"): + """Find historical periods where a specific metric was at a similar level. + + Returns forward returns for those periods, analogous to the composite-score + current_context but filtered to a single metric's historical similarity. + + margin: absolute tolerance for "similar" (auto-computed from metric scale if 0). + """ + try: + from backtesting.engine import run_backtest, HISTORY_PATH, _build_daily_index, _get_all_dates, _last_known_value, METRIC_SCORERS, RATIO_SCORERS, DRAWDOWN_RANGES, _score_range + import os as _os + + if not _os.path.exists(HISTORY_PATH): + return JSONResponse({"error": "No historical data. Run history collector first."}, status_code=404) + + with open(HISTORY_PATH) as f: + history = json.load(f) + + index = _build_daily_index(history) + all_dates = _get_all_dates(index) + + # Get current metric value from cache + cache = {} + if _os.path.exists(CACHE_PATH): + with open(CACHE_PATH) as f: + cache = json.load(f) + + current_raw = _get_current_metric_raw(metric, cache) + if current_raw is None: + return JSONResponse({"error": f"No current value for metric '{metric}'"}, status_code=404) + + # Auto-compute margin from metric scale + if margin <= 0: + margin = _auto_metric_margin(metric, current_raw) + + # Build price lookup + price_lookup = {} + for pk in ["btc_price_coingecko", "btc_price", "btc_price_sma", "btc_price_lth"]: + if pk in index: + for d, v in index[pk].items(): + if d not in price_lookup: + price_lookup[d] = v + + # Find historical days where this metric was similar + comparable = [] + for d in all_dates: + raw_val = _get_historical_metric_raw(metric, index, d) + if raw_val is not None and abs(raw_val - current_raw) <= margin: + price = price_lookup.get(d) + fwd = _compute_day_forward_returns(price_lookup, d) + if fwd: + comparable.append({ + "date": d, + "raw_value": round(raw_val, 6) if isinstance(raw_val, float) else raw_val, + "price": price, + "forward_returns": fwd, + }) + + # Compute average returns across comparable periods + avg_returns = {} + for period in ["30d", "90d", "180d", "365d"]: + vals = [c["forward_returns"][period] for c in comparable if period in c["forward_returns"]] + if vals: + avg_returns[period] = round(sum(vals) / len(vals), 2) + + # Pick best examples (one per market cycle) + cycle_bins = [ + ("pre-2016", "2010-01-01", "2015-12-31"), + ("2016-17 Bull", "2016-01-01", "2017-12-31"), + ("2018-19 Bear", "2018-01-01", "2019-12-31"), + ("2020-21 Bull", "2020-01-01", "2021-12-31"), + ("2022-23 Bear", "2022-01-01", "2023-12-31"), + ("2024+", "2024-01-01", "2099-12-31"), + ] + examples = [] + used_cycles = set() + sorted_comp = sorted(comparable, key=lambda c: abs(c["raw_value"] - current_raw)) + for c in sorted_comp: + for label, start, end in cycle_bins: + if start <= c["date"] <= end and label not in used_cycles: + used_cycles.add(label) + examples.append({ + "date": c["date"], + "raw_value": c["raw_value"], + "price": c["price"], + "forward_returns": c["forward_returns"], + "cycle": label, + }) + break + if len(examples) >= 6: + break + examples.sort(key=lambda e: e["date"]) + + # Percentile: what % of all days had this metric at or below current value + all_raw_vals = [] + for d in all_dates: + rv = _get_historical_metric_raw(metric, index, d) + if rv is not None: + all_raw_vals.append(rv) + all_raw_vals.sort() + below = len([v for v in all_raw_vals if v <= current_raw]) + percentile = round(below / len(all_raw_vals) * 100, 1) if all_raw_vals else 50.0 + + return { + "metric": metric, + "current_raw": current_raw, + "margin": margin, + "comparable_days": len(comparable), + "percentile": percentile, + "avg_30d_return": avg_returns.get("30d"), + "avg_90d_return": avg_returns.get("90d"), + "avg_180d_return": avg_returns.get("180d"), + "avg_1yr_return": avg_returns.get("365d"), + "examples": examples, + } + except Exception as e: + log.error("Metric context error: %s", traceback.format_exc()) + return JSONResponse({"error": str(e)}, status_code=500) + + +def _get_current_metric_raw(metric, cache): + """Get the current raw value for a metric from the cache.""" + # Direct cache keys + direct_keys = { + "fear_greed": ("fear_greed", "value"), + "puell_multiple": ("puell_multiple", "value"), + "mvrv_zscore": ("mvrv_zscore", "value"), + "reserve_risk": ("reserve_risk", "value"), + "rhodl_ratio": ("rhodl_ratio", "value"), + "nupl": ("nupl", "value"), + "drawdown": ("drawdown", "value"), + "hash_ribbons": ("hash_ribbons", "value"), + "sopr": ("sopr", "value"), + "sellside_risk": ("sellside_risk", "value"), + "active_address_momentum": ("active_address_momentum", "value"), + "txcount_momentum": ("txcount_momentum", "value"), + "nvt_price": ("nvt_price", "value"), + "vdd_multiple": ("vdd_multiple", "value"), + "lth_supply": ("lth_supply", "value"), + } + # Ratio-based metrics: compute from price vs reference + ratio_metrics = { + "price_vs_200w_sma": ("price", "200w_sma"), + "lth_realized_price": ("price", "lth_realized_price"), + } + + if metric in direct_keys: + k, sub = direct_keys[metric] + val = cache.get(k, {}) + if isinstance(val, dict): + return val.get(sub) + return val + elif metric in ratio_metrics: + price_key, ref_key = ratio_metrics[metric] + price_val = cache.get(price_key, {}).get("price") or cache.get(price_key, {}).get("value") + ref_val = cache.get(ref_key, {}).get("value") + if price_val and ref_val and ref_val > 0: + return ((price_val - ref_val) / ref_val) * 100 + return None + + +def _get_historical_metric_raw(metric, index, date): + """Get the raw value for a metric on a specific historical date.""" + from backtesting.engine import _last_known_value + direct_keys = { + "fear_greed": "fear_greed", + "puell_multiple": "puell_multiple", + "mvrv_zscore": "mvrv_zscore", + "reserve_risk": "reserve_risk", + "rhodl_ratio": "rhodl_ratio", + "nupl": "nupl", + "drawdown": "drawdown", + "hash_ribbons": "hash_ribbons", + "sopr": "sopr", + "sellside_risk": "sellside_risk", + "active_address_momentum": "active_address_momentum", + "txcount_momentum": "txcount_momentum", + "nvt_price": "nvt_price", + "vdd_multiple": "vdd_multiple", + "lth_supply": "lth_supply", + } + if metric in direct_keys: + return _last_known_value(index.get(direct_keys[metric], {}), date) + # Ratio-based + if metric == "price_vs_200w_sma": + price_val = _last_known_value(index.get("btc_price", {}), date) + ref_val = _last_known_value(index.get("200w_sma", {}), date) + if price_val and ref_val and ref_val > 0: + return ((price_val - ref_val) / ref_val) * 100 + if metric == "lth_realized_price": + price_val = _last_known_value(index.get("btc_price", {}), date) + ref_val = _last_known_value(index.get("lth_realized_price", {}), date) + if price_val and ref_val and ref_val > 0: + return ((price_val - ref_val) / ref_val) * 100 + return None + + +def _auto_metric_margin(metric, current_val): + """Compute a reasonable similarity margin based on metric type and scale.""" + margins = { + "fear_greed": 5.0, + "puell_multiple": 0.15, + "mvrv_zscore": 0.5, + "reserve_risk": 0.002, + "rhodl_ratio": 300, + "nupl": 0.1, + "drawdown": 8.0, + "sopr": 0.02, + "sellside_risk": 0.001, + "active_address_momentum": 0.05, + "txcount_momentum": 0.05, + "nvt_price": 5000, + "vdd_multiple": 0.15, + "price_vs_200w_sma": 10.0, + "lth_realized_price": 10.0, + } + if metric in margins: + return margins[metric] + # Fallback: 15% of current value + return abs(current_val) * 0.15 if current_val != 0 else 1.0 + + +def _compute_day_forward_returns(price_lookup, date): + """Compute forward returns for a single date.""" + from datetime import datetime as _dt, timedelta as _td + p0 = price_lookup.get(date) + if p0 is None or p0 <= 0: + return {} + r = {} + dt = _dt.strptime(date, "%Y-%m-%d") + for days in [30, 90, 180, 365]: + future = (dt + _td(days=days)).strftime("%Y-%m-%d") + pf = price_lookup.get(future) + if pf is not None: + r[f"{days}d"] = round(((pf - p0) / p0) * 100, 2) + return r + + # ── Backtest HTML Page ───────────────────────────────────────────────── BACKTEST_HTML = """ diff --git a/scoring/engine.py b/scoring/engine.py index 1d40074..8ee41df 100644 --- a/scoring/engine.py +++ b/scoring/engine.py @@ -259,6 +259,65 @@ def score_hash_ribbons(data, thresholds=None): return 3, "Normal mining activity" +def score_sopr(value, thresholds=None): + if value is None: + return None, "No data" + if value < 0.98: + return 10, "Deep loss realization — capitulation, strong accumulation" + if value < 1.0: + return 8, "Below breakeven — capitulation, good accumulation" + if value < 1.02: + return 5, "Near breakeven — neutral" + if value < 1.05: + return 2, "Moderate profit taking" + return 0, "Elevated profit taking — caution" + + +def score_sellside_risk(value, thresholds=None): + if value is None: + return None, "No data" + if value < 0.001: + return 10, "Very low sell-side risk — strong accumulation" + if value < 0.002: + return 8, "Low sell-side risk — good accumulation" + if value < 0.005: + return 5, "Moderate sell-side risk" + if value < 0.01: + return 2, "Elevated sell-side risk" + return 0, "High sell-side risk" + + +def score_momentum_pct(value): + if value is None: + return None, "No data" + pct = value * 100 + if pct >= 20: + return 10, f"Strong positive momentum (+{pct:.0f}%)" + if pct >= 0: + return 6, f"Mild positive momentum (+{pct:.0f}%)" + if pct >= -10: + return 4, f"Slightly negative momentum ({pct:.0f}%)" + if pct >= -25: + return 2, f"Weak momentum ({pct:.0f}%)" + return 1, f"Strong negative momentum ({pct:.0f}%)" + + +def score_nvt_price(nvt_price, spot_price): + if nvt_price is None or spot_price is None or spot_price <= 0: + return None, "No data" + premium = (nvt_price - spot_price) / spot_price * 100 + if premium < -25: + return 10, f"NVT price {abs(premium):.0f}% below spot — deep value" + if premium < -10: + return 8, f"NVT price {abs(premium):.0f}% below spot — undervalued" + if premium < 10: + relation = "below" if premium < 0 else "above" + return 5, f"NVT price {abs(premium):.0f}% {relation} spot — fair value" + if premium < 30: + return 2, f"NVT price {premium:.0f}% above spot — extended" + return 0, f"NVT price {premium:.0f}% above spot — overheated" + + def score_all(metrics): """Score all metrics and return individual + composite scores.""" thresholds = load_thresholds() @@ -399,6 +458,84 @@ def score_all(metrics): "recent": [], }) + # SOPR + sopr = metrics.get("sopr", {}) + sopr_score, sopr_desc = score_sopr(sopr.get("value"), thresholds) + results.append({ + "name": "SOPR", + "key": "sopr", + "value": sopr.get("value"), + "display_value": f"{sopr.get('value', 'N/A'):.4f}" if sopr.get("value") is not None else "N/A", + "score": sopr_score, + "description": sopr_desc, + "recent": sopr.get("recent", []), + }) + + # Sell-side Risk Ratio + ssr = metrics.get("sellside_risk", {}) + ssr_score, ssr_desc = score_sellside_risk(ssr.get("value"), thresholds) + results.append({ + "name": "Sell-side Risk Ratio", + "key": "sellside_risk", + "value": ssr.get("value"), + "display_value": f"{ssr.get('value', 'N/A'):.6f}" if ssr.get("value") is not None else "N/A", + "score": ssr_score, + "description": ssr_desc, + "recent": ssr.get("recent", []), + }) + + # Active Address Momentum + aam = metrics.get("active_address_momentum", {}) + aam_score, aam_desc = score_momentum_pct(aam.get("value")) + results.append({ + "name": "Active Address Momentum", + "key": "active_address_momentum", + "value": aam.get("value"), + "display_value": f"{aam.get('value') * 100:.1f}%" if aam.get("value") is not None else "N/A", + "score": aam_score, + "description": aam_desc, + "recent": aam.get("recent", []), + }) + + # Transaction Count Momentum + txm = metrics.get("txcount_momentum", {}) + txm_score, txm_desc = score_momentum_pct(txm.get("value")) + results.append({ + "name": "Transaction Count Momentum", + "key": "txcount_momentum", + "value": txm.get("value"), + "display_value": f"{txm.get('value') * 100:.1f}%" if txm.get("value") is not None else "N/A", + "score": txm_score, + "description": txm_desc, + "recent": txm.get("recent", []), + }) + + # NVT Price + nvt = metrics.get("nvt_price", {}) + nvt_score, nvt_desc = score_nvt_price(nvt.get("value"), current_price) + results.append({ + "name": "NVT Price", + "key": "nvt_price", + "value": nvt.get("value"), + "display_value": f"${nvt.get('value'):,.0f}" if nvt.get("value") is not None else "N/A", + "score": nvt_score, + "description": nvt_desc, + "recent": nvt.get("recent", []), + }) + + # VDD Multiple + vdd = metrics.get("vdd_multiple", {}) + vdd_score, vdd_desc = score_momentum_pct(vdd.get("value")) + results.append({ + "name": "VDD Multiple", + "key": "vdd_multiple", + "value": vdd.get("value"), + "display_value": f"{vdd.get('value') * 100:.1f}%" if vdd.get("value") is not None else "N/A", + "score": vdd_score, + "description": vdd_desc, + "recent": vdd.get("recent", []), + }) + # Compute composite valid_scores = [r["score"] for r in results if r["score"] is not None] if valid_scores: @@ -481,31 +618,34 @@ def score_all_ml(metrics): results = classic["metrics"] - # Compute ML-weighted composite - weighted_sum = 0.0 - weight_total = 0.0 - + # Compute raw ML weights first, then normalize across only the currently + # scored metrics. This keeps the dashboard's displayed per-metric weights and + # contribution points consistent with the normalized composite score even + # when optional metrics are missing or hash ribbons receives its fallback. + weighted_metrics = [] for m in results: if m["score"] is None: continue ml_key = _ML_KEY_MAP.get(m["key"]) if ml_key is None: # Hash ribbons or unknown metric — use small default weight - w = 0.01 + raw_weight = 0.01 else: - w = ml_weights.get(ml_key, 0.0) + raw_weight = ml_weights.get(ml_key, 0.0) + weighted_metrics.append((m, raw_weight)) - m["ml_weight"] = round(w, 4) - m["ml_contribution"] = round(m["score"] * w * 10, 2) - weighted_sum += m["score"] * w - weight_total += w - - # Normalize if weights don't sum to 1 (e.g., missing metrics) + weight_total = sum(raw_weight for _, raw_weight in weighted_metrics) if weight_total > 0: - composite = weighted_sum / weight_total * 10 + composite = sum(m["score"] * raw_weight for m, raw_weight in weighted_metrics) / weight_total * 10 else: composite = 0 + for m, raw_weight in weighted_metrics: + effective_weight = raw_weight / weight_total if weight_total > 0 else 0.0 + m["ml_raw_weight"] = round(raw_weight, 4) + m["ml_weight"] = round(effective_weight, 4) + m["ml_contribution"] = round(m["score"] * effective_weight * 10, 2) + # Assessment text (same thresholds as classic) if composite >= 80: assessment = "EXTREME ACCUMULATION ZONE" @@ -528,4 +668,5 @@ def score_all_ml(metrics): "total_count": classic["total_count"], "ml_mode": True, "classic_score": classic["composite_score"], + "ml_weight_total": round(weight_total, 4), } diff --git a/scrapers/checkonchain.py b/scrapers/checkonchain.py new file mode 100644 index 0000000..570c0a5 --- /dev/null +++ b/scrapers/checkonchain.py @@ -0,0 +1,170 @@ +"""Scraper for static CheckOnChain Plotly chart HTML pages.""" + +from __future__ import annotations + +import array +import base64 +import json +import logging +import re +from html import unescape + +import requests + +log = logging.getLogger(__name__) + +CHARTS = { + "sopr": { + "url": "https://charts.checkonchain.com/btconchain/realised/sopr/sopr_light.html", + "traces": ["SOPR"], + }, + "sellside_risk": { + "url": "https://charts.checkonchain.com/btconchain/realised/sellsideriskratio_all/sellsideriskratio_all_light.html", + "traces": ["Sell-side Risk Ratio", "Sellside Risk Ratio", "SSR"], + }, + "active_address_momentum": { + "url": "https://charts.checkonchain.com/btconchain/adoption/actaddress_momentum/actaddress_momentum_light.html", + "traces": ["30DMA", "30 Day", "Active Address"], + }, + "txcount_momentum": { + "url": "https://charts.checkonchain.com/btconchain/adoption/txcount_momentum/txcount_momentum_light.html", + "traces": ["30DMA", "30 Day", "Transaction"], + }, + "nvt_price": { + "url": "https://charts.checkonchain.com/btconchain/pricing/pricing_nvtprice/pricing_nvtprice_light.html", + "traces": ["NVT Price", "NVT"], + }, + "vdd_multiple": { + "url": "https://charts.checkonchain.com/btconchain/lifespan/vddmultiple/vddmultiple_light.html", + "traces": ["VDD Multiple", "Value Days Destroyed"], + }, +} + + +def _extract_plotly_traces(html_text: str): + """Extract first Plotly.newPlot trace array from a static Plotly HTML page.""" + marker = "Plotly.newPlot(" + start = html_text.find(marker) + if start < 0: + return [] + first_array = html_text.find("[", start) + if first_array < 0: + return [] + + depth = 0 + in_string = False + escape = False + quote = "" + for idx in range(first_array, len(html_text)): + ch = html_text[idx] + if in_string: + if escape: + escape = False + elif ch == "\\": + escape = True + elif ch == quote: + in_string = False + continue + if ch in {'"', "'"}: + in_string = True + quote = ch + elif ch == "[": + depth += 1 + elif ch == "]": + depth -= 1 + if depth == 0: + raw = html_text[first_array:idx + 1] + return json.loads(raw) + return [] + + +def scrape_chart(url: str, timeout=30): + resp = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=timeout) + resp.raise_for_status() + return _extract_plotly_traces(unescape(resp.text)) + + +def _find_trace(traces, names): + names = [n.lower() for n in names if n] + # Prefer non-price traces with the requested terms. + for trace in traces: + trace_name = str(trace.get("name", "")).lower() + if "price" in trace_name and not any("price" in n for n in names): + continue + if any(n in trace_name for n in names): + return trace + # Fallback: first numeric non-price trace. + for trace in traces: + trace_name = str(trace.get("name", "")).lower() + if "price" in trace_name: + continue + y = trace.get("y") or [] + if any(v is not None for v in y[-30:]): + return trace + return None + + +def _decode_plotly_array(values): + """Decode Plotly typed-array JSON ({dtype, bdata}) or return plain values.""" + if not isinstance(values, dict) or "bdata" not in values: + return values or [] + + dtype = values.get("dtype") + typecodes = { + "f8": "d", "float64": "d", + "f4": "f", "float32": "f", + "i8": "q", "int64": "q", + "i4": "i", "int32": "i", + "u8": "Q", "uint64": "Q", + "u4": "I", "uint32": "I", + } + typecode = typecodes.get(dtype) + if not typecode: + return [] + decoded = base64.b64decode(values["bdata"]) + arr = array.array(typecode) + arr.frombytes(decoded) + if values.get("byteorder") == "big": + arr.byteswap() + return arr.tolist() + + +def _numeric_values(trace): + values = [] + for value in _decode_plotly_array((trace or {}).get("y", [])): + if value is None: + continue + try: + values.append(float(value)) + except (TypeError, ValueError): + pass + return values + + +def _latest(values): + return values[-1] if values else None + + +def _momentum(values, window=30): + if len(values) <= window or values[-window] == 0: + return None + return (values[-1] - values[-window]) / values[-window] + + +def scrape_all(): + results = {} + for key, cfg in CHARTS.items(): + log.info("Scraping CheckOnChain %s ...", key) + try: + traces = scrape_chart(cfg["url"]) + trace = _find_trace(traces, cfg.get("traces", [])) + values = _numeric_values(trace) + value = _latest(values) + if key in {"active_address_momentum", "txcount_momentum", "vdd_multiple"}: + # The card value is momentum, while the sparkline shows the raw metric. + value = _momentum(values) + results[key] = {"value": value, "recent": values[-30:]} + except Exception as exc: + log.error("CheckOnChain scrape failed for %s: %s", key, exc) + results[key] = {"value": None, "error": str(exc)} + return results