fix: preserve metrics with atomic persistence

2026-07-26 23:07:30 +00:00
parent 661579abf9
commit 3b1bc9a2bf
4 changed files with 512 additions and 46 deletions
@@ -0,0 +1,99 @@
+import json
+import threading
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+from dashboard.persistence import (
+    append_daily_jsonl,
+    atomic_write_json,
+    load_jsonl_tail,
+    merge_observation,
+    onchain_refresh_due,
+)
+
+
+def test_atomic_write_json_remains_readable_under_concurrent_writers(tmp_path):
+    path = tmp_path / "cache.json"
+
+    threads = [
+        threading.Thread(target=atomic_write_json, args=(path, {"writer": i, "values": list(range(100))}))
+        for i in range(12)
+    ]
+    for thread in threads:
+        thread.start()
+    for thread in threads:
+        thread.join()
+
+    saved = json.loads(path.read_text())
+    assert saved["writer"] in range(12)
+    assert saved["values"] == list(range(100))
+    assert not list(tmp_path.glob(".cache.json.*.tmp"))
+
+
+def test_append_daily_jsonl_writes_at_most_one_entry_per_utc_day(tmp_path):
+    path = tmp_path / "scores.jsonl"
+    first = {"timestamp": "2026-07-26T01:00:00+00:00", "score": 10}
+    duplicate_day = {"timestamp": "2026-07-26T23:59:00+00:00", "score": 20}
+    next_day = {"timestamp": "2026-07-27T00:01:00+00:00", "score": 30}
+
+    assert append_daily_jsonl(path, first) is True
+    assert append_daily_jsonl(path, duplicate_day) is False
+    assert append_daily_jsonl(path, next_day) is True
+
+    assert load_jsonl_tail(path, limit=90) == [first, next_day]
+
+
+def test_load_jsonl_tail_is_bounded_and_ignores_malformed_lines(tmp_path):
+    path = tmp_path / "scores.jsonl"
+    path.write_text("".join(json.dumps({"n": i}) + "\n" for i in range(200)) + "partial{")
+
+    assert load_jsonl_tail(path, limit=3, chunk_size=64) == [{"n": 197}, {"n": 198}, {"n": 199}]
+
+
+def test_merge_observation_preserves_last_known_good_with_stale_metadata():
+    old = {
+        "value": 1.25,
+        "observed_at": "2026-07-25T12:00:00+00:00",
+        "source": "lookintobitcoin",
+        "stale": False,
+        "last_error": None,
+    }
+
+    merged = merge_observation(old, None, source="lookintobitcoin", error="timeout")
+
+    assert merged == {
+        "value": 1.25,
+        "observed_at": "2026-07-25T12:00:00+00:00",
+        "source": "lookintobitcoin",
+        "stale": True,
+        "last_error": "timeout",
+    }
+
+
+def test_merge_observation_records_metadata_for_fresh_value():
+    observed_at = "2026-07-26T12:00:00+00:00"
+
+    merged = merge_observation(
+        {"value": 1.0}, {"value": 2.0, "trend": "up"},
+        source="checkonchain", observed_at=observed_at,
+    )
+
+    assert merged["value"] == 2.0
+    assert merged["trend"] == "up"
+    assert merged["observed_at"] == observed_at
+    assert merged["source"] == "checkonchain"
+    assert merged["stale"] is False
+    assert merged["last_error"] is None
+
+
+@pytest.mark.parametrize("timestamp", [None, "", "not-a-time"])
+def test_onchain_refresh_due_when_timestamp_is_missing_or_invalid(timestamp):
+    assert onchain_refresh_due(timestamp, now=datetime(2026, 7, 26, tzinfo=timezone.utc)) is True
+
+
+def test_onchain_refresh_due_after_ttl():
+    now = datetime(2026, 7, 26, 12, tzinfo=timezone.utc)
+
+    assert onchain_refresh_due((now - timedelta(hours=5)).isoformat(), now=now, ttl_seconds=21600) is False
+    assert onchain_refresh_due((now - timedelta(hours=7)).isoformat(), now=now, ttl_seconds=21600) is True
@@ -0,0 +1,119 @@
+import importlib
+import json
+import sys
+import types
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+
+@pytest.fixture
+def server(monkeypatch):
+    started = []
+    monkeypatch.setattr("threading.Thread.start", lambda self: started.append(self))
+    sys.modules.pop("dashboard.server", None)
+    module = importlib.import_module("dashboard.server")
+    module._threads_started_during_import = started
+    return module
+
+
+def test_server_import_does_not_start_scheduler_threads(server):
+    assert server._threads_started_during_import == []
+
+
+def test_server_cache_and_history_use_reliable_persistence(server, monkeypatch, tmp_path):
+    monkeypatch.setattr(server, "CACHE_PATH", str(tmp_path / "cache.json"))
+    monkeypatch.setattr(server, "HISTORY_PATH", str(tmp_path / "scores.jsonl"))
+
+    server.save_cache({"metric": {"value": 1}})
+    server.append_history({
+        "composite_score": 50,
+        "scored_count": 1,
+        "metrics": [{"key": "metric", "score": 5, "value": 1}],
+    })
+    server.append_history({
+        "composite_score": 60,
+        "scored_count": 1,
+        "metrics": [{"key": "metric", "score": 6, "value": 2}],
+    })
+
+    assert server.load_cache() == {"metric": {"value": 1}}
+    assert len(server.load_history()) == 1
+
+
+def test_partial_fast_scrape_preserves_last_known_good_metric(server, monkeypatch, tmp_path):
+    cache_path = tmp_path / "cache.json"
+    history_path = tmp_path / "scores.jsonl"
+    now = datetime.now(timezone.utc)
+    cache_path.write_text(json.dumps({
+        "price": {
+            "price": 65000,
+            "observed_at": (now - timedelta(minutes=15)).isoformat(),
+            "source": "coingecko",
+            "stale": False,
+            "last_error": None,
+        },
+        "puell_multiple": {"value": 1.2},
+        "_onchain_timestamp": now.isoformat(),
+    }))
+    monkeypatch.setattr(server, "CACHE_PATH", str(cache_path))
+    monkeypatch.setattr(server, "HISTORY_PATH", str(history_path))
+    monkeypatch.setattr(server.fear_greed, "fetch", lambda: {"value": 25})
+    monkeypatch.setattr(server.price, "fetch_current", lambda: (_ for _ in ()).throw(RuntimeError("price timeout")))
+    monkeypatch.setattr(server.price, "fetch_ath", lambda: {"ath": 70000})
+    monkeypatch.setattr(server.price, "fetch_historical", lambda: [])
+    monkeypatch.setattr(server.engine, "score_all", lambda metrics: {"composite_score": 50, "scored_count": 1, "metrics": []})
+    monkeypatch.setattr(server.engine, "score_all_ml", lambda metrics: {"composite_score": 50, "scored_count": 1, "metrics": []})
+    fake_updater = types.ModuleType("scrapers.history_updater")
+    fake_updater.update_history = lambda: None
+    monkeypatch.setitem(sys.modules, "scrapers.history_updater", fake_updater)
+
+    server.run_scrape()
+
+    saved = json.loads(cache_path.read_text())
+    assert saved["price"]["price"] == 65000
+    assert saved["price"]["stale"] is True
+    assert "price timeout" in saved["price"]["last_error"]
+    assert saved["fear_greed"]["value"] == 25
+    assert saved["fear_greed"]["stale"] is False
+    assert saved["fear_greed"]["source"] == "alternative.me"
+
+
+def test_expired_onchain_timestamp_triggers_real_refresh(server, monkeypatch, tmp_path):
+    old = datetime.now(timezone.utc) - timedelta(hours=7)
+    cache_path = tmp_path / "cache.json"
+    cache_path.write_text(json.dumps({
+        "puell_multiple": {"value": 1.2},
+        "_onchain_timestamp": old.isoformat(),
+    }))
+    monkeypatch.setattr(server, "CACHE_PATH", str(cache_path))
+    monkeypatch.setattr(server, "HISTORY_PATH", str(tmp_path / "scores.jsonl"))
+    monkeypatch.setattr(server.fear_greed, "fetch", lambda: {"value": 25})
+    monkeypatch.setattr(server.price, "fetch_current", lambda: {"price": 65000})
+    monkeypatch.setattr(server.price, "fetch_ath", lambda: {"ath": 70000})
+    monkeypatch.setattr(server.price, "fetch_historical", lambda: [])
+    monkeypatch.setattr(server.engine, "score_all", lambda metrics: {"composite_score": 50, "scored_count": 1, "metrics": []})
+    monkeypatch.setattr(server.engine, "score_all_ml", lambda metrics: {"composite_score": 50, "scored_count": 1, "metrics": []})
+
+    calls = []
+    fake_lib = types.ModuleType("scrapers.lookintobitcoin")
+    fake_lib.scrape_all = lambda: calls.append("lib") or {"puell_multiple": {"value": 0.8}}
+    fake_coc = types.ModuleType("scrapers.checkonchain")
+    fake_coc.scrape_all = lambda: calls.append("coc") or {"sopr": {"value": 0.99}}
+    fake_updater = types.ModuleType("scrapers.history_updater")
+    fake_updater.update_history = lambda: None
+    import scrapers
+    monkeypatch.setattr(scrapers, "lookintobitcoin", fake_lib, raising=False)
+    monkeypatch.setattr(scrapers, "checkonchain", fake_coc, raising=False)
+    monkeypatch.setitem(sys.modules, "scrapers.lookintobitcoin", fake_lib)
+    monkeypatch.setitem(sys.modules, "scrapers.checkonchain", fake_coc)
+    monkeypatch.setitem(sys.modules, "scrapers.history_updater", fake_updater)
+
+    server.run_scrape()
+
+    assert calls == ["lib", "coc"]
+    saved = json.loads(cache_path.read_text())
+    assert saved["puell_multiple"]["value"] == 0.8
+    assert saved["puell_multiple"]["source"] == "lookintobitcoin"
+    assert saved["sopr"]["source"] == "checkonchain"
+    assert saved["_onchain_timestamp"] != old.isoformat()