perf: reuse browser across chart scrapes

scrapers/lookintobitcoin.py

@@ -2,6 +2,7 @@
 
 import logging
 import traceback
+from contextlib import contextmanager
 
 log = logging.getLogger(__name__)
 
@@ -51,15 +52,26 @@ CHARTS = {
 }
 
 
-def scrape_chart(chart_path, timeout=25000):
+@contextmanager
-    """Scrape a single chart from LookIntoBitcoin. Returns list of trace dicts or None."""
+def browser_page():
+    """Open one headless browser page for a batch of chart requests."""
     from playwright.sync_api import sync_playwright
 
-    store = {"data": None}
+    with sync_playwright() as playwright:
+        browser = playwright.chromium.launch(headless=True)
+        try:
+            yield browser.new_page()
+        finally:
+            browser.close()
 
-    with sync_playwright() as p:
+
-        browser = p.chromium.launch(headless=True)
+def scrape_chart(chart_path, timeout=25000, page=None):
-        page = browser.new_page()
+    """Scrape one chart, optionally reusing a caller-owned browser page."""
+    if page is None:
+        with browser_page() as owned_page:
+            return scrape_chart(chart_path, timeout=timeout, page=owned_page)
+
+    store = {"data": None}
 
     def handle_response(response):
         if "_dash-update-component" in response.url:
@@ -72,10 +84,10 @@ def scrape_chart(chart_path, timeout=25000):
     try:
         page.goto(f"{BASE_URL}{chart_path}", timeout=timeout)
         page.wait_for_timeout(6000)
-        except Exception as e:
+    except Exception as exc:
-            log.warning("Navigation error for %s: %s", chart_path, e)
+        log.warning("Navigation error for %s: %s", chart_path, exc)
     finally:
-            browser.close()
+        page.remove_listener("response", handle_response)
 
     if store["data"]:
         try:
@@ -172,13 +184,18 @@ def _get_recent_values(trace, n=30):
 
 
 def scrape_all():
-    """Scrape all charts and return parsed metric values."""
+    """Scrape all charts while reusing one browser process and page."""
+    with browser_page() as page:
+        return _scrape_all_with_page(page)
+
+
+def _scrape_all_with_page(page):
     results = {}
 
     for metric_key, chart_info in CHARTS.items():
         log.info("Scraping %s ...", metric_key)
         try:
-            traces = scrape_chart(chart_info["path"])
+            traces = scrape_chart(chart_info["path"], page=page)
             if not traces:
                 log.warning("No data for %s", metric_key)
                 results[metric_key] = {"value": None, "error": "No data returned"}

tests/test_scraper_semantics.py

@@ -1,3 +1,5 @@
+from contextlib import contextmanager
+
 from scrapers import lookintobitcoin
 from scoring import engine
 
@@ -26,3 +28,29 @@ def test_vdd_derived_return_is_labeled_as_momentum_not_raw_multiple():
 
     assert vdd["name"] == "VDD 30-Period Momentum"
     assert vdd["transform"] == "30_period_return"
+
+
+def test_scrape_all_reuses_one_browser_page(monkeypatch):
+    page = object()
+    seen_pages = []
+
+    @contextmanager
+    def fake_browser_page():
+        yield page
+
+    def fake_scrape_chart(_path, timeout=25000, page=None):
+        seen_pages.append(page)
+        return [{"name": "metric", "y": [1.0]}]
+
+    monkeypatch.setattr(lookintobitcoin, "CHARTS", {
+        "first": {"path": "/first", "traces": ["metric"]},
+        "second": {"path": "/second", "traces": ["metric"]},
+    })
+    monkeypatch.setattr(lookintobitcoin, "browser_page", fake_browser_page)
+    monkeypatch.setattr(lookintobitcoin, "scrape_chart", fake_scrape_chart)
+
+    result = lookintobitcoin.scrape_all()
+
+    assert seen_pages == [page, page]
+    assert result["first"]["value"] == 1.0
+    assert result["second"]["value"] == 1.0