22fc7fc6cd
- Historical data (5693+ points per metric) saved in history.json permanently - Quick refresh: only updates price + Fear & Greed from APIs (~2 seconds) - Full refresh: only needed for FIRST-TIME setup or if data is missing - Daily append: new values added to history.json from cache, not re-scraped - Startup: uses cached on-chain data if it exists, no unnecessary Playwright launches - On-chain metrics only update once per day, no reason to re-scrape them
113 lines
3.7 KiB
Python
113 lines
3.7 KiB
Python
"""Incremental history updater — appends new daily data to history.json from cache."""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
from datetime import datetime, timezone
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data")
|
|
HISTORY_PATH = os.path.join(DATA_DIR, "history.json")
|
|
CACHE_PATH = os.path.join(DATA_DIR, "cache.json")
|
|
|
|
|
|
def update_history():
|
|
"""Append today's values from cache to history.json. Only adds NEW dates."""
|
|
if not os.path.exists(HISTORY_PATH):
|
|
log.warning("No history.json found — run full collection first")
|
|
return False
|
|
|
|
if not os.path.exists(CACHE_PATH):
|
|
log.warning("No cache.json found — run a scrape first")
|
|
return False
|
|
|
|
with open(HISTORY_PATH) as f:
|
|
history = json.load(f)
|
|
with open(CACHE_PATH) as f:
|
|
cache = json.load(f)
|
|
|
|
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
|
updated = False
|
|
|
|
# Map of cache keys to history keys and how to extract the value
|
|
mappings = {
|
|
"puell_multiple": {"history_key": "puell_multiple", "value_key": "value"},
|
|
"mvrv_zscore": {"history_key": "mvrv_zscore", "value_key": "value"},
|
|
"reserve_risk": {"history_key": "reserve_risk", "value_key": "value"},
|
|
"rhodl_ratio": {"history_key": "rhodl_ratio", "value_key": "value"},
|
|
"nupl": {"history_key": "nupl", "value_key": "value"},
|
|
"200w_sma": {"history_key": "200w_sma", "value_key": "value"},
|
|
"lth_realized_price": {"history_key": "lth_realized_price", "value_key": "value"},
|
|
"lth_supply": {"history_key": "lth_supply", "value_key": "value"},
|
|
}
|
|
|
|
for cache_key, mapping in mappings.items():
|
|
hkey = mapping["history_key"]
|
|
if hkey not in history:
|
|
continue
|
|
|
|
h = history[hkey]
|
|
dates = h.get("dates", [])
|
|
values = h.get("values", [])
|
|
|
|
# Skip if today already in history
|
|
if dates and dates[-1] >= today:
|
|
continue
|
|
|
|
# Get value from cache
|
|
cached = cache.get(cache_key, {})
|
|
val = cached.get(mapping["value_key"])
|
|
if val is not None:
|
|
dates.append(today)
|
|
values.append(val)
|
|
h["dates"] = dates
|
|
h["values"] = values
|
|
updated = True
|
|
log.info("Appended %s: %s = %s", hkey, today, val)
|
|
|
|
# Also update btc_price from cache
|
|
price_data = cache.get("price", {})
|
|
btc_price = price_data.get("price")
|
|
if btc_price and "btc_price" in history:
|
|
h = history["btc_price"]
|
|
if h["dates"][-1] < today:
|
|
h["dates"].append(today)
|
|
h["values"].append(btc_price)
|
|
updated = True
|
|
|
|
# BTC price for SMA chart
|
|
if btc_price and "btc_price_sma" in history:
|
|
h = history["btc_price_sma"]
|
|
if h["dates"][-1] < today:
|
|
h["dates"].append(today)
|
|
h["values"].append(btc_price)
|
|
updated = True
|
|
|
|
# BTC price for LTH chart
|
|
if btc_price and "btc_price_lth" in history:
|
|
h = history["btc_price_lth"]
|
|
if h["dates"][-1] < today:
|
|
h["dates"].append(today)
|
|
h["values"].append(btc_price)
|
|
updated = True
|
|
|
|
# Fear & Greed
|
|
fg = cache.get("fear_greed", {})
|
|
fg_val = fg.get("value")
|
|
if fg_val is not None and "fear_greed" in history:
|
|
h = history["fear_greed"]
|
|
if h["dates"][-1] < today:
|
|
h["dates"].append(today)
|
|
h["values"].append(int(fg_val))
|
|
updated = True
|
|
|
|
if updated:
|
|
with open(HISTORY_PATH, "w") as f:
|
|
json.dump(history, f)
|
|
log.info("History updated with %s data", today)
|
|
else:
|
|
log.info("History already up to date (last date >= %s)", today)
|
|
|
|
return updated
|