fix: historical data stored permanently, only append new daily values

- Historical data (5693+ points per metric) saved in history.json permanently
- Quick refresh: only updates price + Fear & Greed from APIs (~2 seconds)
- Full refresh: only needed for FIRST-TIME setup or if data is missing
- Daily append: new values added to history.json from cache, not re-scraped
- Startup: uses cached on-chain data if it exists, no unnecessary Playwright launches
- On-chain metrics only update once per day, no reason to re-scrape them

dashboard/server.py

@@ -139,28 +139,16 @@ def run_scrape(force_full=False):
             mayer = price.calculate_mayer_multiple(price_current.get("price"), sma_200d)
             metrics["price_extras"] = {"sma_200d": sma_200d, "mayer_multiple": mayer}
 
-        # 3. On-chain metrics via Playwright (slow — only when needed)
+        # 3. On-chain metrics — use cached values (historical data is permanent)
         onchain_keys = ["puell_multiple", "mvrv_zscore", "reserve_risk", "rhodl_ratio",
                         "nupl", "200w_sma", "lth_realized_price", "hash_ribbons",
                         "pi_cycle_bottom", "lth_supply"]
         
-        # Check if we need a full on-chain refresh
-        cached_ts = existing_cache.get("_onchain_timestamp")
-        onchain_stale = True
-        if cached_ts and not force_full:
-            try:
-                from datetime import datetime as dt
-                age_hours = (datetime.now(timezone.utc) - datetime.fromisoformat(cached_ts)).total_seconds() / 3600
-                onchain_stale = age_hours > 6
-                if not onchain_stale:
-                    log.info("On-chain data is %.1fh old — reusing cache (next full refresh in %.1fh)", age_hours, 6 - age_hours)
-            except Exception:
-                onchain_stale = True
-        
         has_cached_onchain = any(existing_cache.get(k, {}).get("value") is not None for k in onchain_keys)
         
-        if force_full or onchain_stale or not has_cached_onchain:
-            log.info("Scraping on-chain metrics from LookIntoBitcoin (full refresh)...")
+        if force_full or not has_cached_onchain:
+            # Only do a full Playwright scrape if explicitly requested or no data exists
+            log.info("Scraping on-chain metrics from LookIntoBitcoin (full refresh requested)...")
             try:
                 from scrapers import lookintobitcoin
                 onchain = lookintobitcoin.scrape_all()
@@ -169,14 +157,12 @@ def run_scrape(force_full=False):
             except Exception as e:
                 log.error("LookIntoBitcoin scraping failed: %s\n%s", e, traceback.format_exc())
                 _last_error = f"On-chain scraping failed: {e}"
-                # Fall back to cached on-chain data
                 for k in onchain_keys:
                     if k in existing_cache:
                         metrics[k] = existing_cache[k]
-                if "_onchain_timestamp" in existing_cache:
-                    metrics["_onchain_timestamp"] = existing_cache["_onchain_timestamp"]
         else:
-            # Reuse cached on-chain data
+            # Reuse cached on-chain values — they're stored permanently
+            log.info("Reusing cached on-chain data (use Full Refresh to re-scrape)")
             for k in onchain_keys:
                 if k in existing_cache:
                     metrics[k] = existing_cache[k]
@@ -192,6 +178,13 @@ def run_scrape(force_full=False):
         save_cache(metrics)
         append_history(scored)
 
+        # Append today's values to permanent history (incremental, not full re-scrape)
+        try:
+            from scrapers.history_updater import update_history
+            update_history()
+        except Exception as e:
+            log.warning("History update failed (non-critical): %s", e)
+
         _last_update = datetime.now(timezone.utc).isoformat()
         _last_error = None
         log.info("Scrape cycle complete. Composite score: %s", scored["composite_score"])
@@ -205,11 +198,14 @@ def run_scrape(force_full=False):
 
 
 def scraper_loop():
-    """Background loop: quick refresh every 15min, full on-chain refresh every 6h."""
-    run_scrape(force_full=True)  # Full scrape on first boot if no cached data
+    """Background loop: quick refresh every 15min. Full scrape only on first boot with no data."""
+    cache = load_cache()
+    has_data = any(cache.get(k, {}).get("value") is not None 
+                   for k in ["puell_multiple", "mvrv_zscore", "nupl"])
+    run_scrape(force_full=not has_data)  # Full only if no cached on-chain data
     while True:
         time.sleep(900)  # 15 minutes
-        run_scrape()  # Quick refresh (reuses cached on-chain if <6h old)
+        run_scrape()  # Quick refresh only
 
 
 # Start background scraper on import