fix: historical data stored permanently, only append new daily values

- Historical data (5693+ points per metric) saved in history.json permanently - Quick refresh: only updates price + Fear & Greed from APIs (~2 seconds) - Full refresh: only needed for FIRST-TIME setup or if data is missing - Daily append: new values added to history.json from cache, not re-scraped - Startup: uses cached on-chain data if it exists, no unnecessary Playwright launches - On-chain metrics only update once per day, no reason to re-scrape them
2026-03-20 23:29:39 +00:00
parent 28b5240a81
commit 22fc7fc6cd
3 changed files with 132 additions and 23 deletions
@@ -0,0 +1,112 @@
+"""Incremental history updater — appends new daily data to history.json from cache."""
+
+import json
+import logging
+import os
+from datetime import datetime, timezone
+
+log = logging.getLogger(__name__)
+
+DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data")
+HISTORY_PATH = os.path.join(DATA_DIR, "history.json")
+CACHE_PATH = os.path.join(DATA_DIR, "cache.json")
+
+
+def update_history():
+    """Append today's values from cache to history.json. Only adds NEW dates."""
+    if not os.path.exists(HISTORY_PATH):
+        log.warning("No history.json found — run full collection first")
+        return False
+
+    if not os.path.exists(CACHE_PATH):
+        log.warning("No cache.json found — run a scrape first")
+        return False
+
+    with open(HISTORY_PATH) as f:
+        history = json.load(f)
+    with open(CACHE_PATH) as f:
+        cache = json.load(f)
+
+    today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    updated = False
+
+    # Map of cache keys to history keys and how to extract the value
+    mappings = {
+        "puell_multiple": {"history_key": "puell_multiple", "value_key": "value"},
+        "mvrv_zscore": {"history_key": "mvrv_zscore", "value_key": "value"},
+        "reserve_risk": {"history_key": "reserve_risk", "value_key": "value"},
+        "rhodl_ratio": {"history_key": "rhodl_ratio", "value_key": "value"},
+        "nupl": {"history_key": "nupl", "value_key": "value"},
+        "200w_sma": {"history_key": "200w_sma", "value_key": "value"},
+        "lth_realized_price": {"history_key": "lth_realized_price", "value_key": "value"},
+        "lth_supply": {"history_key": "lth_supply", "value_key": "value"},
+    }
+
+    for cache_key, mapping in mappings.items():
+        hkey = mapping["history_key"]
+        if hkey not in history:
+            continue
+
+        h = history[hkey]
+        dates = h.get("dates", [])
+        values = h.get("values", [])
+
+        # Skip if today already in history
+        if dates and dates[-1] >= today:
+            continue
+
+        # Get value from cache
+        cached = cache.get(cache_key, {})
+        val = cached.get(mapping["value_key"])
+        if val is not None:
+            dates.append(today)
+            values.append(val)
+            h["dates"] = dates
+            h["values"] = values
+            updated = True
+            log.info("Appended %s: %s = %s", hkey, today, val)
+
+    # Also update btc_price from cache
+    price_data = cache.get("price", {})
+    btc_price = price_data.get("price")
+    if btc_price and "btc_price" in history:
+        h = history["btc_price"]
+        if h["dates"][-1] < today:
+            h["dates"].append(today)
+            h["values"].append(btc_price)
+            updated = True
+
+    # BTC price for SMA chart
+    if btc_price and "btc_price_sma" in history:
+        h = history["btc_price_sma"]
+        if h["dates"][-1] < today:
+            h["dates"].append(today)
+            h["values"].append(btc_price)
+            updated = True
+
+    # BTC price for LTH chart
+    if btc_price and "btc_price_lth" in history:
+        h = history["btc_price_lth"]
+        if h["dates"][-1] < today:
+            h["dates"].append(today)
+            h["values"].append(btc_price)
+            updated = True
+
+    # Fear & Greed
+    fg = cache.get("fear_greed", {})
+    fg_val = fg.get("value")
+    if fg_val is not None and "fear_greed" in history:
+        h = history["fear_greed"]
+        if h["dates"][-1] < today:
+            h["dates"].append(today)
+            h["values"].append(int(fg_val))
+            updated = True
+
+    if updated:
+        with open(HISTORY_PATH, "w") as f:
+            json.dump(history, f)
+        log.info("History updated with %s data", today)
+    else:
+        log.info("History already up to date (last date >= %s)", today)
+
+    return updated