From 28b5240a8154f44ba953b6b2e2eeb9a9c4a6e147 Mon Sep 17 00:00:00 2001 From: BizzleBot Date: Fri, 20 Mar 2026 23:25:54 +0000 Subject: [PATCH] =?UTF-8?q?perf:=20smart=20refresh=20=E2=80=94=20quick=20u?= =?UTF-8?q?pdates=20price/F&G=20only,=20full=20scrape=20every=206h?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Quick Refresh button: updates price + Fear & Greed only (~2 seconds) - Full Refresh button: re-scrapes all on-chain data from LookIntoBitcoin (~2-3 min) - Background auto-refresh: quick every 15min, full only when on-chain data >6h old - Cached on-chain data preserved between quick refreshes - On-chain metrics only update daily anyway, no need to re-scrape every 15min --- dashboard/server.py | 98 ++++++++++++++++++++++++++++++---------- data/score_history.jsonl | 1 + 2 files changed, 74 insertions(+), 25 deletions(-) diff --git a/dashboard/server.py b/dashboard/server.py index df35c62..16d609d 100644 --- a/dashboard/server.py +++ b/dashboard/server.py @@ -94,8 +94,15 @@ def load_history(): # ── Background scraper ──────────────────────────────────────────────────── -def run_scrape(): - """Run a full scrape cycle and update cache.""" +def run_scrape(force_full=False): + """Run a scrape cycle and update cache. + + By default, only refreshes fast data (price, F&G) and reuses cached on-chain data. + On-chain metrics (Playwright scrapes) only refresh if: + - force_full=True (manual full refresh) + - No cached on-chain data exists + - Cached on-chain data is >6 hours old (they update daily) + """ global _last_update, _last_error, _scraper_running with _scraper_lock: @@ -104,7 +111,8 @@ def run_scrape(): _scraper_running = True try: - log.info("Starting scrape cycle...") + # Load existing cache to preserve on-chain data + existing_cache = load_cache() metrics = {} # 1. Fear & Greed (fast API call) @@ -131,15 +139,49 @@ def run_scrape(): mayer = price.calculate_mayer_multiple(price_current.get("price"), sma_200d) metrics["price_extras"] = {"sma_200d": sma_200d, "mayer_multiple": mayer} - # 3. On-chain metrics via Playwright (slow) - log.info("Scraping on-chain metrics from LookIntoBitcoin...") - try: - from scrapers import lookintobitcoin - onchain = lookintobitcoin.scrape_all() - metrics.update(onchain) - except Exception as e: - log.error("LookIntoBitcoin scraping failed: %s\n%s", e, traceback.format_exc()) - _last_error = f"On-chain scraping failed: {e}" + # 3. On-chain metrics via Playwright (slow — only when needed) + onchain_keys = ["puell_multiple", "mvrv_zscore", "reserve_risk", "rhodl_ratio", + "nupl", "200w_sma", "lth_realized_price", "hash_ribbons", + "pi_cycle_bottom", "lth_supply"] + + # Check if we need a full on-chain refresh + cached_ts = existing_cache.get("_onchain_timestamp") + onchain_stale = True + if cached_ts and not force_full: + try: + from datetime import datetime as dt + age_hours = (datetime.now(timezone.utc) - datetime.fromisoformat(cached_ts)).total_seconds() / 3600 + onchain_stale = age_hours > 6 + if not onchain_stale: + log.info("On-chain data is %.1fh old — reusing cache (next full refresh in %.1fh)", age_hours, 6 - age_hours) + except Exception: + onchain_stale = True + + has_cached_onchain = any(existing_cache.get(k, {}).get("value") is not None for k in onchain_keys) + + if force_full or onchain_stale or not has_cached_onchain: + log.info("Scraping on-chain metrics from LookIntoBitcoin (full refresh)...") + try: + from scrapers import lookintobitcoin + onchain = lookintobitcoin.scrape_all() + metrics.update(onchain) + metrics["_onchain_timestamp"] = datetime.now(timezone.utc).isoformat() + except Exception as e: + log.error("LookIntoBitcoin scraping failed: %s\n%s", e, traceback.format_exc()) + _last_error = f"On-chain scraping failed: {e}" + # Fall back to cached on-chain data + for k in onchain_keys: + if k in existing_cache: + metrics[k] = existing_cache[k] + if "_onchain_timestamp" in existing_cache: + metrics["_onchain_timestamp"] = existing_cache["_onchain_timestamp"] + else: + # Reuse cached on-chain data + for k in onchain_keys: + if k in existing_cache: + metrics[k] = existing_cache[k] + if "_onchain_timestamp" in existing_cache: + metrics["_onchain_timestamp"] = existing_cache["_onchain_timestamp"] # 4. Score everything log.info("Scoring metrics...") @@ -163,10 +205,11 @@ def run_scrape(): def scraper_loop(): - """Background loop that runs scrape every 15 minutes.""" + """Background loop: quick refresh every 15min, full on-chain refresh every 6h.""" + run_scrape(force_full=True) # Full scrape on first boot if no cached data while True: - run_scrape() time.sleep(900) # 15 minutes + run_scrape() # Quick refresh (reuses cached on-chain if <6h old) # Start background scraper on import @@ -299,13 +342,15 @@ def api_history(): @app.post("/api/refresh") -def api_refresh(): - """Trigger a manual scrape.""" +def api_refresh(full: bool = False): + """Trigger a scrape. Quick refresh (default) updates price + F&G only (~2s). + Full refresh (?full=true) also re-scrapes on-chain data via Playwright (~2-3min).""" if _scraper_running: return JSONResponse({"error": "Scrape already in progress"}, status_code=409) - t = threading.Thread(target=run_scrape, daemon=True) + t = threading.Thread(target=run_scrape, kwargs={"force_full": full}, daemon=True) t.start() - return {"ok": True, "message": "Scrape started"} + mode = "full (on-chain + price + F&G)" if full else "quick (price + F&G only)" + return {"ok": True, "message": f"Scrape started — {mode}"} # Settings routes (preserved) @@ -463,7 +508,8 @@ DASHBOARD_HTML = """ Loading... - + + @@ -740,17 +786,19 @@ async function poll() { } catch(e) { console.error('Poll error:', e); } } -async function doRefresh() { - const btn = document.getElementById('btnRefresh'); +async function doRefresh(full) { + const btn = document.getElementById(full ? 'btnFullRefresh' : 'btnRefresh'); + const origText = btn.textContent; btn.disabled = true; - btn.textContent = 'Refreshing...'; + btn.textContent = full ? 'Scraping...' : 'Refreshing...'; try { - const r = await fetch('/api/refresh', { method: 'POST' }); + const r = await fetch('/api/refresh' + (full ? '?full=true' : ''), { method: 'POST' }); const d = await r.json(); if (d.error) showToast(d.error, 'error'); - else showToast('Scrape started — results in ~2 min', 'success'); + else showToast(d.message || 'Refresh started', 'success'); } catch(e) { showToast('Failed: ' + e, 'error'); } - setTimeout(() => { btn.disabled = false; btn.textContent = 'Refresh Data'; }, 5000); + const delay = full ? 180000 : 5000; + setTimeout(() => { btn.disabled = false; btn.textContent = origText; }, delay); } drawScoreRing(0); diff --git a/data/score_history.jsonl b/data/score_history.jsonl index 57f906c..cbd87da 100644 --- a/data/score_history.jsonl +++ b/data/score_history.jsonl @@ -3,3 +3,4 @@ {"timestamp": "2026-03-20T22:46:34.952569+00:00", "composite_score": 51.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 43.931630710659896}, "price_vs_200w_sma": {"score": 3, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}} {"timestamp": "2026-03-20T22:51:27.724327+00:00", "composite_score": 54.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 43.94907994923858}, "price_vs_200w_sma": {"score": 6, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}} {"timestamp": "2026-03-20T23:07:48.303808+00:00", "composite_score": 51.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 43.942734771573605}, "price_vs_200w_sma": {"score": 3, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}} +{"timestamp": "2026-03-20T23:21:39.705718+00:00", "composite_score": 54.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 44.07439720812183}, "price_vs_200w_sma": {"score": 6, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}}