From 1e50760f27ccda64cc719c8257836710fe49f85f Mon Sep 17 00:00:00 2001
From: Hermes Agent <hermes@nousresearch.com>
Date: Sun, 26 Jul 2026 23:27:06 +0000
Subject: [PATCH] perf: reuse browser across chart scrapes

---
 scrapers/lookintobitcoin.py     | 61 +++++++++++++++++++++------------
 tests/test_scraper_semantics.py | 28 +++++++++++++++
 2 files changed, 67 insertions(+), 22 deletions(-)

diff --git a/scrapers/lookintobitcoin.py b/scrapers/lookintobitcoin.py
index 5706120..3dd685a 100644
--- a/scrapers/lookintobitcoin.py
+++ b/scrapers/lookintobitcoin.py
@@ -2,6 +2,7 @@
 
 import logging
 import traceback
+from contextlib import contextmanager
 
 log = logging.getLogger(__name__)
 
@@ -51,31 +52,42 @@ CHARTS = {
 }
 
 
-def scrape_chart(chart_path, timeout=25000):
-    """Scrape a single chart from LookIntoBitcoin. Returns list of trace dicts or None."""
+@contextmanager
+def browser_page():
+    """Open one headless browser page for a batch of chart requests."""
     from playwright.sync_api import sync_playwright
 
+    with sync_playwright() as playwright:
+        browser = playwright.chromium.launch(headless=True)
+        try:
+            yield browser.new_page()
+        finally:
+            browser.close()
+
+
+def scrape_chart(chart_path, timeout=25000, page=None):
+    """Scrape one chart, optionally reusing a caller-owned browser page."""
+    if page is None:
+        with browser_page() as owned_page:
+            return scrape_chart(chart_path, timeout=timeout, page=owned_page)
+
     store = {"data": None}
 
-    with sync_playwright() as p:
-        browser = p.chromium.launch(headless=True)
-        page = browser.new_page()
+    def handle_response(response):
+        if "_dash-update-component" in response.url:
+            try:
+                store["data"] = response.json()
+            except Exception:
+                pass
 
-        def handle_response(response):
-            if "_dash-update-component" in response.url:
-                try:
-                    store["data"] = response.json()
-                except Exception:
-                    pass
-
-        page.on("response", handle_response)
-        try:
-            page.goto(f"{BASE_URL}{chart_path}", timeout=timeout)
-            page.wait_for_timeout(6000)
-        except Exception as e:
-            log.warning("Navigation error for %s: %s", chart_path, e)
-        finally:
-            browser.close()
+    page.on("response", handle_response)
+    try:
+        page.goto(f"{BASE_URL}{chart_path}", timeout=timeout)
+        page.wait_for_timeout(6000)
+    except Exception as exc:
+        log.warning("Navigation error for %s: %s", chart_path, exc)
+    finally:
+        page.remove_listener("response", handle_response)
 
     if store["data"]:
         try:
@@ -172,13 +184,18 @@ def _get_recent_values(trace, n=30):
 
 
 def scrape_all():
-    """Scrape all charts and return parsed metric values."""
+    """Scrape all charts while reusing one browser process and page."""
+    with browser_page() as page:
+        return _scrape_all_with_page(page)
+
+
+def _scrape_all_with_page(page):
     results = {}
 
     for metric_key, chart_info in CHARTS.items():
         log.info("Scraping %s ...", metric_key)
         try:
-            traces = scrape_chart(chart_info["path"])
+            traces = scrape_chart(chart_info["path"], page=page)
             if not traces:
                 log.warning("No data for %s", metric_key)
                 results[metric_key] = {"value": None, "error": "No data returned"}
diff --git a/tests/test_scraper_semantics.py b/tests/test_scraper_semantics.py
index 1db9c78..08f2ac0 100644
--- a/tests/test_scraper_semantics.py
+++ b/tests/test_scraper_semantics.py
@@ -1,3 +1,5 @@
+from contextlib import contextmanager
+
 from scrapers import lookintobitcoin
 from scoring import engine
 
@@ -26,3 +28,29 @@ def test_vdd_derived_return_is_labeled_as_momentum_not_raw_multiple():
 
     assert vdd["name"] == "VDD 30-Period Momentum"
     assert vdd["transform"] == "30_period_return"
+
+
+def test_scrape_all_reuses_one_browser_page(monkeypatch):
+    page = object()
+    seen_pages = []
+
+    @contextmanager
+    def fake_browser_page():
+        yield page
+
+    def fake_scrape_chart(_path, timeout=25000, page=None):
+        seen_pages.append(page)
+        return [{"name": "metric", "y": [1.0]}]
+
+    monkeypatch.setattr(lookintobitcoin, "CHARTS", {
+        "first": {"path": "/first", "traces": ["metric"]},
+        "second": {"path": "/second", "traces": ["metric"]},
+    })
+    monkeypatch.setattr(lookintobitcoin, "browser_page", fake_browser_page)
+    monkeypatch.setattr(lookintobitcoin, "scrape_chart", fake_scrape_chart)
+
+    result = lookintobitcoin.scrape_all()
+
+    assert seen_pages == [page, page]
+    assert result["first"]["value"] == 1.0
+    assert result["second"]["value"] == 1.0