perf: smart refresh — quick updates price/F&G only, full scrape every 6h

- Quick Refresh button: updates price + Fear & Greed only (~2 seconds)
- Full Refresh button: re-scrapes all on-chain data from LookIntoBitcoin (~2-3 min)
- Background auto-refresh: quick every 15min, full only when on-chain data >6h old
- Cached on-chain data preserved between quick refreshes
- On-chain metrics only update daily anyway, no need to re-scrape every 15min

dashboard/server.py

@@ -94,8 +94,15 @@ def load_history():
 
 # ── Background scraper ────────────────────────────────────────────────────
 
-def run_scrape():
+def run_scrape(force_full=False):
-    """Run a full scrape cycle and update cache."""
+    """Run a scrape cycle and update cache.
+    
+    By default, only refreshes fast data (price, F&G) and reuses cached on-chain data.
+    On-chain metrics (Playwright scrapes) only refresh if:
+    - force_full=True (manual full refresh)
+    - No cached on-chain data exists
+    - Cached on-chain data is >6 hours old (they update daily)
+    """
     global _last_update, _last_error, _scraper_running
 
     with _scraper_lock:
@@ -104,7 +111,8 @@ def run_scrape():
         _scraper_running = True
 
     try:
-        log.info("Starting scrape cycle...")
+        # Load existing cache to preserve on-chain data
+        existing_cache = load_cache()
         metrics = {}
 
         # 1. Fear & Greed (fast API call)
@@ -131,15 +139,49 @@ def run_scrape():
             mayer = price.calculate_mayer_multiple(price_current.get("price"), sma_200d)
             metrics["price_extras"] = {"sma_200d": sma_200d, "mayer_multiple": mayer}
 
-        # 3. On-chain metrics via Playwright (slow)
+        # 3. On-chain metrics via Playwright (slow — only when needed)
-        log.info("Scraping on-chain metrics from LookIntoBitcoin...")
+        onchain_keys = ["puell_multiple", "mvrv_zscore", "reserve_risk", "rhodl_ratio",
-        try:
+                        "nupl", "200w_sma", "lth_realized_price", "hash_ribbons",
-            from scrapers import lookintobitcoin
+                        "pi_cycle_bottom", "lth_supply"]
-            onchain = lookintobitcoin.scrape_all()
+        
-            metrics.update(onchain)
+        # Check if we need a full on-chain refresh
-        except Exception as e:
+        cached_ts = existing_cache.get("_onchain_timestamp")
-            log.error("LookIntoBitcoin scraping failed: %s\n%s", e, traceback.format_exc())
+        onchain_stale = True
-            _last_error = f"On-chain scraping failed: {e}"
+        if cached_ts and not force_full:
+            try:
+                from datetime import datetime as dt
+                age_hours = (datetime.now(timezone.utc) - datetime.fromisoformat(cached_ts)).total_seconds() / 3600
+                onchain_stale = age_hours > 6
+                if not onchain_stale:
+                    log.info("On-chain data is %.1fh old — reusing cache (next full refresh in %.1fh)", age_hours, 6 - age_hours)
+            except Exception:
+                onchain_stale = True
+        
+        has_cached_onchain = any(existing_cache.get(k, {}).get("value") is not None for k in onchain_keys)
+        
+        if force_full or onchain_stale or not has_cached_onchain:
+            log.info("Scraping on-chain metrics from LookIntoBitcoin (full refresh)...")
+            try:
+                from scrapers import lookintobitcoin
+                onchain = lookintobitcoin.scrape_all()
+                metrics.update(onchain)
+                metrics["_onchain_timestamp"] = datetime.now(timezone.utc).isoformat()
+            except Exception as e:
+                log.error("LookIntoBitcoin scraping failed: %s\n%s", e, traceback.format_exc())
+                _last_error = f"On-chain scraping failed: {e}"
+                # Fall back to cached on-chain data
+                for k in onchain_keys:
+                    if k in existing_cache:
+                        metrics[k] = existing_cache[k]
+                if "_onchain_timestamp" in existing_cache:
+                    metrics["_onchain_timestamp"] = existing_cache["_onchain_timestamp"]
+        else:
+            # Reuse cached on-chain data
+            for k in onchain_keys:
+                if k in existing_cache:
+                    metrics[k] = existing_cache[k]
+            if "_onchain_timestamp" in existing_cache:
+                metrics["_onchain_timestamp"] = existing_cache["_onchain_timestamp"]
 
         # 4. Score everything
         log.info("Scoring metrics...")
@@ -163,10 +205,11 @@ def run_scrape():
 
 
 def scraper_loop():
-    """Background loop that runs scrape every 15 minutes."""
+    """Background loop: quick refresh every 15min, full on-chain refresh every 6h."""
+    run_scrape(force_full=True)  # Full scrape on first boot if no cached data
     while True:
-        run_scrape()
         time.sleep(900)  # 15 minutes
+        run_scrape()  # Quick refresh (reuses cached on-chain if <6h old)
 
 
 # Start background scraper on import
@@ -299,13 +342,15 @@ def api_history():
 
 
 @app.post("/api/refresh")
-def api_refresh():
+def api_refresh(full: bool = False):
-    """Trigger a manual scrape."""
+    """Trigger a scrape. Quick refresh (default) updates price + F&G only (~2s).
+    Full refresh (?full=true) also re-scrapes on-chain data via Playwright (~2-3min)."""
     if _scraper_running:
         return JSONResponse({"error": "Scrape already in progress"}, status_code=409)
-    t = threading.Thread(target=run_scrape, daemon=True)
+    t = threading.Thread(target=run_scrape, kwargs={"force_full": full}, daemon=True)
     t.start()
-    return {"ok": True, "message": "Scrape started"}
+    mode = "full (on-chain + price + F&G)" if full else "quick (price + F&G only)"
+    return {"ok": True, "message": f"Scrape started — {mode}"}
 
 
 # Settings routes (preserved)
@@ -463,7 +508,8 @@ DASHBOARD_HTML = """<!DOCTYPE html>
       <span class="status-dot" id="statusDot"></span>
       <span id="statusText">Loading...</span>
     </div>
-    <button class="btn btn-accent" onclick="doRefresh()" id="btnRefresh">Refresh Data</button>
+    <button class="btn btn-accent" onclick="doRefresh(false)" id="btnRefresh">⚡ Quick Refresh</button>
+    <button class="btn btn-secondary" onclick="doRefresh(true)" id="btnFullRefresh" title="Re-scrape on-chain metrics from LookIntoBitcoin (~2-3 min)">🔄 Full Refresh</button>
   </div>
 </div>
 
@@ -740,17 +786,19 @@ async function poll() {
   } catch(e) { console.error('Poll error:', e); }
 }
 
-async function doRefresh() {
+async function doRefresh(full) {
-  const btn = document.getElementById('btnRefresh');
+  const btn = document.getElementById(full ? 'btnFullRefresh' : 'btnRefresh');
+  const origText = btn.textContent;
   btn.disabled = true;
-  btn.textContent = 'Refreshing...';
+  btn.textContent = full ? 'Scraping...' : 'Refreshing...';
   try {
-    const r = await fetch('/api/refresh', { method: 'POST' });
+    const r = await fetch('/api/refresh' + (full ? '?full=true' : ''), { method: 'POST' });
     const d = await r.json();
     if (d.error) showToast(d.error, 'error');
-    else showToast('Scrape started — results in ~2 min', 'success');
+    else showToast(d.message || 'Refresh started', 'success');
   } catch(e) { showToast('Failed: ' + e, 'error'); }
-  setTimeout(() => { btn.disabled = false; btn.textContent = 'Refresh Data'; }, 5000);
+  const delay = full ? 180000 : 5000;
+  setTimeout(() => { btn.disabled = false; btn.textContent = origText; }, delay);
 }
 
 drawScoreRing(0);

data/score_history.jsonl

@@ -3,3 +3,4 @@
 {"timestamp": "2026-03-20T22:46:34.952569+00:00", "composite_score": 51.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 43.931630710659896}, "price_vs_200w_sma": {"score": 3, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}}
 {"timestamp": "2026-03-20T22:51:27.724327+00:00", "composite_score": 54.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 43.94907994923858}, "price_vs_200w_sma": {"score": 6, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}}
 {"timestamp": "2026-03-20T23:07:48.303808+00:00", "composite_score": 51.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 43.942734771573605}, "price_vs_200w_sma": {"score": 3, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}}
+{"timestamp": "2026-03-20T23:21:39.705718+00:00", "composite_score": 54.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 44.07439720812183}, "price_vs_200w_sma": {"score": 6, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}}