perf: smart refresh — quick updates price/F&G only, full scrape every 6h
- Quick Refresh button: updates price + Fear & Greed only (~2 seconds) - Full Refresh button: re-scrapes all on-chain data from LookIntoBitcoin (~2-3 min) - Background auto-refresh: quick every 15min, full only when on-chain data >6h old - Cached on-chain data preserved between quick refreshes - On-chain metrics only update daily anyway, no need to re-scrape every 15min
This commit is contained in:
BizzleBot
parent
e385765fda
commit
28b5240a81
@ -94,8 +94,15 @@ def load_history():
|
||||
|
||||
# ── Background scraper ────────────────────────────────────────────────────
|
||||
|
||||
def run_scrape():
|
||||
"""Run a full scrape cycle and update cache."""
|
||||
def run_scrape(force_full=False):
|
||||
"""Run a scrape cycle and update cache.
|
||||
|
||||
By default, only refreshes fast data (price, F&G) and reuses cached on-chain data.
|
||||
On-chain metrics (Playwright scrapes) only refresh if:
|
||||
- force_full=True (manual full refresh)
|
||||
- No cached on-chain data exists
|
||||
- Cached on-chain data is >6 hours old (they update daily)
|
||||
"""
|
||||
global _last_update, _last_error, _scraper_running
|
||||
|
||||
with _scraper_lock:
|
||||
@ -104,7 +111,8 @@ def run_scrape():
|
||||
_scraper_running = True
|
||||
|
||||
try:
|
||||
log.info("Starting scrape cycle...")
|
||||
# Load existing cache to preserve on-chain data
|
||||
existing_cache = load_cache()
|
||||
metrics = {}
|
||||
|
||||
# 1. Fear & Greed (fast API call)
|
||||
@ -131,15 +139,49 @@ def run_scrape():
|
||||
mayer = price.calculate_mayer_multiple(price_current.get("price"), sma_200d)
|
||||
metrics["price_extras"] = {"sma_200d": sma_200d, "mayer_multiple": mayer}
|
||||
|
||||
# 3. On-chain metrics via Playwright (slow)
|
||||
log.info("Scraping on-chain metrics from LookIntoBitcoin...")
|
||||
try:
|
||||
from scrapers import lookintobitcoin
|
||||
onchain = lookintobitcoin.scrape_all()
|
||||
metrics.update(onchain)
|
||||
except Exception as e:
|
||||
log.error("LookIntoBitcoin scraping failed: %s\n%s", e, traceback.format_exc())
|
||||
_last_error = f"On-chain scraping failed: {e}"
|
||||
# 3. On-chain metrics via Playwright (slow — only when needed)
|
||||
onchain_keys = ["puell_multiple", "mvrv_zscore", "reserve_risk", "rhodl_ratio",
|
||||
"nupl", "200w_sma", "lth_realized_price", "hash_ribbons",
|
||||
"pi_cycle_bottom", "lth_supply"]
|
||||
|
||||
# Check if we need a full on-chain refresh
|
||||
cached_ts = existing_cache.get("_onchain_timestamp")
|
||||
onchain_stale = True
|
||||
if cached_ts and not force_full:
|
||||
try:
|
||||
from datetime import datetime as dt
|
||||
age_hours = (datetime.now(timezone.utc) - datetime.fromisoformat(cached_ts)).total_seconds() / 3600
|
||||
onchain_stale = age_hours > 6
|
||||
if not onchain_stale:
|
||||
log.info("On-chain data is %.1fh old — reusing cache (next full refresh in %.1fh)", age_hours, 6 - age_hours)
|
||||
except Exception:
|
||||
onchain_stale = True
|
||||
|
||||
has_cached_onchain = any(existing_cache.get(k, {}).get("value") is not None for k in onchain_keys)
|
||||
|
||||
if force_full or onchain_stale or not has_cached_onchain:
|
||||
log.info("Scraping on-chain metrics from LookIntoBitcoin (full refresh)...")
|
||||
try:
|
||||
from scrapers import lookintobitcoin
|
||||
onchain = lookintobitcoin.scrape_all()
|
||||
metrics.update(onchain)
|
||||
metrics["_onchain_timestamp"] = datetime.now(timezone.utc).isoformat()
|
||||
except Exception as e:
|
||||
log.error("LookIntoBitcoin scraping failed: %s\n%s", e, traceback.format_exc())
|
||||
_last_error = f"On-chain scraping failed: {e}"
|
||||
# Fall back to cached on-chain data
|
||||
for k in onchain_keys:
|
||||
if k in existing_cache:
|
||||
metrics[k] = existing_cache[k]
|
||||
if "_onchain_timestamp" in existing_cache:
|
||||
metrics["_onchain_timestamp"] = existing_cache["_onchain_timestamp"]
|
||||
else:
|
||||
# Reuse cached on-chain data
|
||||
for k in onchain_keys:
|
||||
if k in existing_cache:
|
||||
metrics[k] = existing_cache[k]
|
||||
if "_onchain_timestamp" in existing_cache:
|
||||
metrics["_onchain_timestamp"] = existing_cache["_onchain_timestamp"]
|
||||
|
||||
# 4. Score everything
|
||||
log.info("Scoring metrics...")
|
||||
@ -163,10 +205,11 @@ def run_scrape():
|
||||
|
||||
|
||||
def scraper_loop():
|
||||
"""Background loop that runs scrape every 15 minutes."""
|
||||
"""Background loop: quick refresh every 15min, full on-chain refresh every 6h."""
|
||||
run_scrape(force_full=True) # Full scrape on first boot if no cached data
|
||||
while True:
|
||||
run_scrape()
|
||||
time.sleep(900) # 15 minutes
|
||||
run_scrape() # Quick refresh (reuses cached on-chain if <6h old)
|
||||
|
||||
|
||||
# Start background scraper on import
|
||||
@ -299,13 +342,15 @@ def api_history():
|
||||
|
||||
|
||||
@app.post("/api/refresh")
|
||||
def api_refresh():
|
||||
"""Trigger a manual scrape."""
|
||||
def api_refresh(full: bool = False):
|
||||
"""Trigger a scrape. Quick refresh (default) updates price + F&G only (~2s).
|
||||
Full refresh (?full=true) also re-scrapes on-chain data via Playwright (~2-3min)."""
|
||||
if _scraper_running:
|
||||
return JSONResponse({"error": "Scrape already in progress"}, status_code=409)
|
||||
t = threading.Thread(target=run_scrape, daemon=True)
|
||||
t = threading.Thread(target=run_scrape, kwargs={"force_full": full}, daemon=True)
|
||||
t.start()
|
||||
return {"ok": True, "message": "Scrape started"}
|
||||
mode = "full (on-chain + price + F&G)" if full else "quick (price + F&G only)"
|
||||
return {"ok": True, "message": f"Scrape started — {mode}"}
|
||||
|
||||
|
||||
# Settings routes (preserved)
|
||||
@ -463,7 +508,8 @@ DASHBOARD_HTML = """<!DOCTYPE html>
|
||||
<span class="status-dot" id="statusDot"></span>
|
||||
<span id="statusText">Loading...</span>
|
||||
</div>
|
||||
<button class="btn btn-accent" onclick="doRefresh()" id="btnRefresh">Refresh Data</button>
|
||||
<button class="btn btn-accent" onclick="doRefresh(false)" id="btnRefresh">⚡ Quick Refresh</button>
|
||||
<button class="btn btn-secondary" onclick="doRefresh(true)" id="btnFullRefresh" title="Re-scrape on-chain metrics from LookIntoBitcoin (~2-3 min)">🔄 Full Refresh</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -740,17 +786,19 @@ async function poll() {
|
||||
} catch(e) { console.error('Poll error:', e); }
|
||||
}
|
||||
|
||||
async function doRefresh() {
|
||||
const btn = document.getElementById('btnRefresh');
|
||||
async function doRefresh(full) {
|
||||
const btn = document.getElementById(full ? 'btnFullRefresh' : 'btnRefresh');
|
||||
const origText = btn.textContent;
|
||||
btn.disabled = true;
|
||||
btn.textContent = 'Refreshing...';
|
||||
btn.textContent = full ? 'Scraping...' : 'Refreshing...';
|
||||
try {
|
||||
const r = await fetch('/api/refresh', { method: 'POST' });
|
||||
const r = await fetch('/api/refresh' + (full ? '?full=true' : ''), { method: 'POST' });
|
||||
const d = await r.json();
|
||||
if (d.error) showToast(d.error, 'error');
|
||||
else showToast('Scrape started — results in ~2 min', 'success');
|
||||
else showToast(d.message || 'Refresh started', 'success');
|
||||
} catch(e) { showToast('Failed: ' + e, 'error'); }
|
||||
setTimeout(() => { btn.disabled = false; btn.textContent = 'Refresh Data'; }, 5000);
|
||||
const delay = full ? 180000 : 5000;
|
||||
setTimeout(() => { btn.disabled = false; btn.textContent = origText; }, delay);
|
||||
}
|
||||
|
||||
drawScoreRing(0);
|
||||
|
||||
@ -3,3 +3,4 @@
|
||||
{"timestamp": "2026-03-20T22:46:34.952569+00:00", "composite_score": 51.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 43.931630710659896}, "price_vs_200w_sma": {"score": 3, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}}
|
||||
{"timestamp": "2026-03-20T22:51:27.724327+00:00", "composite_score": 54.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 43.94907994923858}, "price_vs_200w_sma": {"score": 6, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}}
|
||||
{"timestamp": "2026-03-20T23:07:48.303808+00:00", "composite_score": 51.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 43.942734771573605}, "price_vs_200w_sma": {"score": 3, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}}
|
||||
{"timestamp": "2026-03-20T23:21:39.705718+00:00", "composite_score": 54.0, "scored_count": 10, "metrics": {"fear_greed": {"score": 7, "value": 11}, "puell_multiple": {"score": 5, "value": 0.6602699608966011}, "mvrv_zscore": {"score": 5, "value": 0.5211180167687892}, "drawdown": {"score": 6, "value": 44.07439720812183}, "price_vs_200w_sma": {"score": 6, "value": 58895.78086828114}, "reserve_risk": {"score": 10, "value": 0.0012985709697654493}, "rhodl_ratio": {"score": 4, "value": 1230.6243545314708}, "nupl": {"score": 7, "value": 0.22243290955405431}, "lth_realized_price": {"score": 1, "value": 43346.58756410873}, "hash_ribbons": {"score": 3, "value": null}}}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user