diff --git a/dashboard/server.py b/dashboard/server.py
index df35c62..16d609d 100644
--- a/dashboard/server.py
+++ b/dashboard/server.py
@@ -94,8 +94,15 @@ def load_history():
# ── Background scraper ────────────────────────────────────────────────────
-def run_scrape():
- """Run a full scrape cycle and update cache."""
+def run_scrape(force_full=False):
+ """Run a scrape cycle and update cache.
+
+ By default, only refreshes fast data (price, F&G) and reuses cached on-chain data.
+ On-chain metrics (Playwright scrapes) only refresh if:
+ - force_full=True (manual full refresh)
+ - No cached on-chain data exists
+ - Cached on-chain data is >6 hours old (they update daily)
+ """
global _last_update, _last_error, _scraper_running
with _scraper_lock:
@@ -104,7 +111,8 @@ def run_scrape():
_scraper_running = True
try:
- log.info("Starting scrape cycle...")
+ # Load existing cache to preserve on-chain data
+ existing_cache = load_cache()
metrics = {}
# 1. Fear & Greed (fast API call)
@@ -131,15 +139,49 @@ def run_scrape():
mayer = price.calculate_mayer_multiple(price_current.get("price"), sma_200d)
metrics["price_extras"] = {"sma_200d": sma_200d, "mayer_multiple": mayer}
- # 3. On-chain metrics via Playwright (slow)
- log.info("Scraping on-chain metrics from LookIntoBitcoin...")
- try:
- from scrapers import lookintobitcoin
- onchain = lookintobitcoin.scrape_all()
- metrics.update(onchain)
- except Exception as e:
- log.error("LookIntoBitcoin scraping failed: %s\n%s", e, traceback.format_exc())
- _last_error = f"On-chain scraping failed: {e}"
+ # 3. On-chain metrics via Playwright (slow — only when needed)
+ onchain_keys = ["puell_multiple", "mvrv_zscore", "reserve_risk", "rhodl_ratio",
+ "nupl", "200w_sma", "lth_realized_price", "hash_ribbons",
+ "pi_cycle_bottom", "lth_supply"]
+
+ # Check if we need a full on-chain refresh
+ cached_ts = existing_cache.get("_onchain_timestamp")
+ onchain_stale = True
+ if cached_ts and not force_full:
+ try:
+ from datetime import datetime as dt
+ age_hours = (datetime.now(timezone.utc) - datetime.fromisoformat(cached_ts)).total_seconds() / 3600
+ onchain_stale = age_hours > 6
+ if not onchain_stale:
+ log.info("On-chain data is %.1fh old — reusing cache (next full refresh in %.1fh)", age_hours, 6 - age_hours)
+ except Exception:
+ onchain_stale = True
+
+ has_cached_onchain = any(existing_cache.get(k, {}).get("value") is not None for k in onchain_keys)
+
+ if force_full or onchain_stale or not has_cached_onchain:
+ log.info("Scraping on-chain metrics from LookIntoBitcoin (full refresh)...")
+ try:
+ from scrapers import lookintobitcoin
+ onchain = lookintobitcoin.scrape_all()
+ metrics.update(onchain)
+ metrics["_onchain_timestamp"] = datetime.now(timezone.utc).isoformat()
+ except Exception as e:
+ log.error("LookIntoBitcoin scraping failed: %s\n%s", e, traceback.format_exc())
+ _last_error = f"On-chain scraping failed: {e}"
+ # Fall back to cached on-chain data
+ for k in onchain_keys:
+ if k in existing_cache:
+ metrics[k] = existing_cache[k]
+ if "_onchain_timestamp" in existing_cache:
+ metrics["_onchain_timestamp"] = existing_cache["_onchain_timestamp"]
+ else:
+ # Reuse cached on-chain data
+ for k in onchain_keys:
+ if k in existing_cache:
+ metrics[k] = existing_cache[k]
+ if "_onchain_timestamp" in existing_cache:
+ metrics["_onchain_timestamp"] = existing_cache["_onchain_timestamp"]
# 4. Score everything
log.info("Scoring metrics...")
@@ -163,10 +205,11 @@ def run_scrape():
def scraper_loop():
- """Background loop that runs scrape every 15 minutes."""
+ """Background loop: quick refresh every 15min, full on-chain refresh every 6h."""
+ run_scrape(force_full=True) # Full scrape on first boot if no cached data
while True:
- run_scrape()
time.sleep(900) # 15 minutes
+ run_scrape() # Quick refresh (reuses cached on-chain if <6h old)
# Start background scraper on import
@@ -299,13 +342,15 @@ def api_history():
@app.post("/api/refresh")
-def api_refresh():
- """Trigger a manual scrape."""
+def api_refresh(full: bool = False):
+ """Trigger a scrape. Quick refresh (default) updates price + F&G only (~2s).
+ Full refresh (?full=true) also re-scrapes on-chain data via Playwright (~2-3min)."""
if _scraper_running:
return JSONResponse({"error": "Scrape already in progress"}, status_code=409)
- t = threading.Thread(target=run_scrape, daemon=True)
+ t = threading.Thread(target=run_scrape, kwargs={"force_full": full}, daemon=True)
t.start()
- return {"ok": True, "message": "Scrape started"}
+ mode = "full (on-chain + price + F&G)" if full else "quick (price + F&G only)"
+ return {"ok": True, "message": f"Scrape started — {mode}"}
# Settings routes (preserved)
@@ -463,7 +508,8 @@ DASHBOARD_HTML = """
Loading...
-
+
+
@@ -740,17 +786,19 @@ async function poll() {
} catch(e) { console.error('Poll error:', e); }
}
-async function doRefresh() {
- const btn = document.getElementById('btnRefresh');
+async function doRefresh(full) {
+ const btn = document.getElementById(full ? 'btnFullRefresh' : 'btnRefresh');
+ const origText = btn.textContent;
btn.disabled = true;
- btn.textContent = 'Refreshing...';
+ btn.textContent = full ? 'Scraping...' : 'Refreshing...';
try {
- const r = await fetch('/api/refresh', { method: 'POST' });
+ const r = await fetch('/api/refresh' + (full ? '?full=true' : ''), { method: 'POST' });
const d = await r.json();
if (d.error) showToast(d.error, 'error');
- else showToast('Scrape started — results in ~2 min', 'success');
+ else showToast(d.message || 'Refresh started', 'success');
} catch(e) { showToast('Failed: ' + e, 'error'); }
- setTimeout(() => { btn.disabled = false; btn.textContent = 'Refresh Data'; }, 5000);
+ const delay = full ? 180000 : 5000;
+ setTimeout(() => { btn.disabled = false; btn.textContent = origText; }, delay);
}
drawScoreRing(0);
diff --git a/data/score_history.jsonl b/data/score_history.jsonl
index 57f906c..cbd87da 100644
--- a/data/score_history.jsonl
+++ b/data/score_history.jsonl
@@ -3,3 +3,4 @@
{"timestamp": "2026-03-20T22:46:34.952569+00:00", "composite_score": 51.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 43.931630710659896}, "price_vs_200w_sma": {"score": 3, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}}
{"timestamp": "2026-03-20T22:51:27.724327+00:00", "composite_score": 54.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 43.94907994923858}, "price_vs_200w_sma": {"score": 6, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}}
{"timestamp": "2026-03-20T23:07:48.303808+00:00", "composite_score": 51.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 43.942734771573605}, "price_vs_200w_sma": {"score": 3, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}}
+{"timestamp": "2026-03-20T23:21:39.705718+00:00", "composite_score": 54.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 44.07439720812183}, "price_vs_200w_sma": {"score": 6, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}}