bizzle/btc-accumulation-monitor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pivot: rewrite as BTC accumulation signal optimizer

Browse Source 
Replace day-trading bot with long-term accumulation signal model.
Predicts optimal BUY times using forward return analysis at 7d/30d/90d
horizons, scoring each candle 0-100. Primary metric is now
cost_basis_improvement_pct (model buy price vs DCA).

- train_and_backtest.py: regression models (XGBoost/LSTM hybrid),
  accumulation-focused features (price position, momentum, volatility,
  volume, cycle), forward return targets, signal quality backtesting
- orchestrator.py: cost improvement scoring, signal count validation
- analyzer.py: accumulation-focused LLM system prompt
- dashboard: cost improvement display, signal metrics table
- config: new accumulation-focused parameters

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
BizzleBot 2026-03-19 23:51:43 +00:00
parent a21e635d9f
commit 560863fa0d
5 changed files with 829 additions and 797 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
config/initial_config.json
View File

@ -1,62 +1,53 @@
{ {
"model_type": "hybrid", "model_type": "hybrid",
"features": { "features": {
"technical_indicators": ["RSI_14", "RSI_7", "MACD_line", "MACD_signal", "MACD_hist", "BB_upper", "BB_lower", "BB_width", "ATR_14", "SMA_20", "SMA_50", "EMA_10", "EMA_20", "OBV", "stoch_k", "stoch_d", "williams_r", "CCI_20", "ROC_10"], "use_price_position": true,
"lookback_periods": [3, 5, 10, 20], "use_momentum": true,
"use_volume_features": true, "use_volatility": true,
"use_volatility_features": true, "use_volume": true,
"use_candle_patterns": false, "use_cycle": true,
"use_lag_features": true,
"lag_periods": [1, 2, 3, 5],
"use_pca": true, "use_pca": true,
"pca_variance": 0.95, "pca_variance": 0.95,
"use_scaler": true "use_scaler": true
}, },
"target": { "target": {
"type": "classification", "type": "regression",
"direction": "both", "forward_periods_1h": [168, 720, 2160],
"horizon_candles": 8, "forward_periods_4h": [42, 180, 540],
"threshold_pct": 1.5 "weights": [0.2, 0.3, 0.5],
"score_range": [0, 100]
}, },
"hyperparameters": { "hyperparameters": {
"learning_rate": 0.001, "learning_rate": 0.01,
"max_depth": 5, "max_depth": 5,
"n_estimators": 300, "n_estimators": 500,
"subsample": 0.8, "subsample": 0.8,
"colsample_bytree": 0.8, "colsample_bytree": 0.8,
"min_child_weight": 5, "min_child_weight": 10,
"gamma": 0.3, "gamma": 0.3,
"reg_alpha": 0.1, "reg_alpha": 0.5,
"reg_lambda": 2.0, "reg_lambda": 3.0,
"lstm_hidden_size": 128, "lstm_hidden_size": 128,
"lstm_num_layers": 2, "lstm_num_layers": 2,
"lstm_dropout": 0.3, "lstm_dropout": 0.3,
"lstm_epochs": 100, "lstm_epochs": 100,
"lstm_batch_size": 64, "lstm_batch_size": 64,
"lstm_sequence_length": 20, "lstm_sequence_length": 30,
"lstm_patience": 10 "lstm_patience": 10
}, },
"strategy": { "strategy": {
"entry_threshold": 0.60, "strong_buy_threshold": 80,
"exit_type": "trailing_stop", "good_buy_threshold": 70,
"stop_loss_pct": 2.0, "poor_threshold": 30
"take_profit_pct": 4.0,
"trailing_stop_pct": 1.5,
"position_sizing": "confidence_scaled",
"max_position_pct": 100,
"min_confidence_to_trade": 0.55,
"dynamic_sl_tp": true,
"atr_sl_multiplier": 1.5,
"atr_tp_multiplier": 3.0
}, },
"training": { "training": {
"rolling_window": true,
"rolling_train_size": 2500,
"rolling_test_size": 300,
"walk_forward_windows": 5, "walk_forward_windows": 5,
"train_pct": 0.7, "train_pct": 0.7,
"validation_pct": 0.15, "validation_pct": 0.15,
"test_pct": 0.15, "test_pct": 0.15
"rolling_window": true,
"rolling_train_size": 2000,
"rolling_test_size": 200
}, },
"timeframe": "4h" "timeframe": "4h"
} }

113
dashboard/server.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
BTC ML Trading Strategy Optimizer — Web Dashboard BTC Accumulation Signal Optimizer -- Web Dashboard
FastAPI server with inline HTML/CSS/JS dashboard. FastAPI server with inline HTML/CSS/JS dashboard.
""" """
@ -13,42 +13,35 @@ from fastapi import FastAPI
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
from pydantic import BaseModel from pydantic import BaseModel
# Add project root to path
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, BASE_DIR) sys.path.insert(0, BASE_DIR)
import orchestrator import orchestrator
app = FastAPI(title="BTC ML Optimizer Dashboard") app = FastAPI(title="BTC Accumulation Signal Optimizer")
CONFIG_DIR = os.path.join(BASE_DIR, "config") CONFIG_DIR = os.path.join(BASE_DIR, "config")
RESULTS_DIR = os.path.join(BASE_DIR, "results") RESULTS_DIR = os.path.join(BASE_DIR, "results")
ITERATIONS_LOG = os.path.join(RESULTS_DIR, "iterations.jsonl") ITERATIONS_LOG = os.path.join(RESULTS_DIR, "iterations.jsonl")
# Background thread reference _opt_thread = None
_opt_thread: threading.Thread | None = None
class ConfigUpdate(BaseModel): class ConfigUpdate(BaseModel):
config: dict config: dict
# ── API Endpoints ──────────────────────────────────────────────
@app.get("/api/status") @app.get("/api/status")
def api_status(): def api_status():
status = orchestrator.get_status() return orchestrator.get_status()
return status
@app.get("/api/iterations") @app.get("/api/iterations")
def api_iterations(): def api_iterations():
iterations = orchestrator.load_iteration_history() iterations = orchestrator.load_iteration_history()
# Strip heavy config from list view
slim = [] slim = []
for it in iterations: for it in iterations:
entry = {k: v for k, v in it.items() if k != "config"} entry = {k: v for k, v in it.items() if k not in ("config", "results")}
slim.append(entry) slim.append(entry)
return slim return slim
@ -94,11 +87,11 @@ def api_stop():
def api_best(): def api_best():
best_path = os.path.join(CONFIG_DIR, "best_config.json") best_path = os.path.join(CONFIG_DIR, "best_config.json")
if not os.path.exists(best_path): if not os.path.exists(best_path):
return {"config": None, "sharpe": 0} return {"config": None, "best_score": 0}
with open(best_path) as f: with open(best_path) as f:
config = json.load(f) config = json.load(f)
iterations = orchestrator.load_iteration_history() iterations = orchestrator.load_iteration_history()
best_iter = max(iterations, key=lambda x: x.get("sharpe", 0)) if iterations else {} best_iter = max(iterations, key=lambda x: x.get("cost_improvement", 0)) if iterations else {}
return {"config": config, "best_iteration": best_iter} return {"config": config, "best_iteration": best_iter}
@ -117,15 +110,12 @@ def api_download_best_config():
return JSONResponse({"error": "No best config yet"}, status_code=404) return JSONResponse({"error": "No best config yet"}, status_code=404)
# ── Dashboard HTML ─────────────────────────────────────────────
DASHBOARD_HTML = """<!DOCTYPE html> DASHBOARD_HTML = """<!DOCTYPE html>
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>BTC ML Optimizer</title> <title>BTC Accumulation Signal Optimizer</title>
<link rel="preconnect" href="https://fonts.googleapis.com"> <link rel="preconnect" href="https://fonts.googleapis.com">
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet"> <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet">
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.4/dist/chart.umd.min.js"></script> <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.4/dist/chart.umd.min.js"></script>
@ -138,7 +128,6 @@ h1{font-size:1.5rem;font-weight:700;display:flex;align-items:center;gap:10px}
h1 .btc{color:var(--accent);font-size:1.8rem} h1 .btc{color:var(--accent);font-size:1.8rem}
h2{font-size:1rem;font-weight:600;color:var(--text-dim);margin-bottom:12px;text-transform:uppercase;letter-spacing:.05em;font-size:.8rem} h2{font-size:1rem;font-weight:600;color:var(--text-dim);margin-bottom:12px;text-transform:uppercase;letter-spacing:.05em;font-size:.8rem}
/* Header */
.header{display:flex;justify-content:space-between;align-items:center;padding:16px 0;border-bottom:1px solid var(--border);margin-bottom:16px;flex-wrap:wrap;gap:12px} .header{display:flex;justify-content:space-between;align-items:center;padding:16px 0;border-bottom:1px solid var(--border);margin-bottom:16px;flex-wrap:wrap;gap:12px}
.controls{display:flex;gap:8px;align-items:center} .controls{display:flex;gap:8px;align-items:center}
.btn{padding:8px 18px;border:none;border-radius:6px;font-family:inherit;font-weight:600;font-size:.85rem;cursor:pointer;transition:all .15s} .btn{padding:8px 18px;border:none;border-radius:6px;font-family:inherit;font-weight:600;font-size:.85rem;cursor:pointer;transition:all .15s}
@ -147,7 +136,6 @@ h2{font-size:1rem;font-weight:600;color:var(--text-dim);margin-bottom:12px;text-
.btn-secondary{background:var(--border);color:var(--text)}.btn-secondary:hover{background:var(--card-hover)} .btn-secondary{background:var(--border);color:var(--text)}.btn-secondary:hover{background:var(--card-hover)}
.btn:disabled{opacity:.4;cursor:not-allowed} .btn:disabled{opacity:.4;cursor:not-allowed}
/* Status badge */
.status-badge{display:inline-flex;align-items:center;gap:6px;padding:4px 12px;border-radius:20px;font-size:.8rem;font-weight:600} .status-badge{display:inline-flex;align-items:center;gap:6px;padding:4px 12px;border-radius:20px;font-size:.8rem;font-weight:600}
.status-idle{background:#1e3a5f;color:#60a5fa} .status-idle{background:#1e3a5f;color:#60a5fa}
.status-running{background:#1a3a2a;color:var(--green)} .status-running{background:#1a3a2a;color:var(--green)}
@ -156,19 +144,16 @@ h2{font-size:1rem;font-weight:600;color:var(--text-dim);margin-bottom:12px;text-
.pulse{width:8px;height:8px;border-radius:50%;background:currentColor;animation:pulse 1.5s infinite} .pulse{width:8px;height:8px;border-radius:50%;background:currentColor;animation:pulse 1.5s infinite}
@keyframes pulse{0%,100%{opacity:1}50%{opacity:.3}} @keyframes pulse{0%,100%{opacity:1}50%{opacity:.3}}
/* Best Sharpe display */ .best-score{text-align:right}
.best-sharpe{text-align:right} .best-score .label{font-size:.7rem;text-transform:uppercase;letter-spacing:.1em;color:var(--text-dim)}
.best-sharpe .label{font-size:.7rem;text-transform:uppercase;letter-spacing:.1em;color:var(--text-dim)} .best-score .value{font-size:2.2rem;font-weight:700;color:var(--accent);font-family:var(--mono)}
.best-sharpe .value{font-size:2.2rem;font-weight:700;color:var(--accent);font-family:var(--mono)} .best-score .unit{font-size:1rem;color:var(--text-dim)}
/* Grid layout */
.grid{display:grid;grid-template-columns:1fr 360px;gap:16px} .grid{display:grid;grid-template-columns:1fr 360px;gap:16px}
@media(max-width:900px){.grid{grid-template-columns:1fr}} @media(max-width:900px){.grid{grid-template-columns:1fr}}
/* Cards */
.card{background:var(--card);border-radius:10px;padding:16px;border:1px solid var(--border)} .card{background:var(--card);border-radius:10px;padding:16px;border:1px solid var(--border)}
/* Iteration table */
.table-wrap{overflow-x:auto;max-height:400px;overflow-y:auto} .table-wrap{overflow-x:auto;max-height:400px;overflow-y:auto}
table{width:100%;border-collapse:collapse;font-size:.82rem} table{width:100%;border-collapse:collapse;font-size:.82rem}
th{position:sticky;top:0;background:var(--card);text-align:left;padding:8px 10px;color:var(--text-dim);font-weight:600;border-bottom:2px solid var(--border);font-size:.75rem;text-transform:uppercase;letter-spacing:.04em} th{position:sticky;top:0;background:var(--card);text-align:left;padding:8px 10px;color:var(--text-dim);font-weight:600;border-bottom:2px solid var(--border);font-size:.75rem;text-transform:uppercase;letter-spacing:.04em}
@ -177,15 +162,12 @@ tr.best-row{background:rgba(34,197,94,.1)}
tr.best-row td:first-child{border-left:3px solid var(--green)} tr.best-row td:first-child{border-left:3px solid var(--green)}
tr:hover{background:var(--card-hover)} tr:hover{background:var(--card-hover)}
/* Chart */
.chart-container{position:relative;height:260px} .chart-container{position:relative;height:260px}
/* LLM panel */
.llm-panel{max-height:500px;overflow-y:auto} .llm-panel{max-height:500px;overflow-y:auto}
.llm-entry{padding:10px;border-bottom:1px solid var(--border);font-size:.82rem;line-height:1.5} .llm-entry{padding:10px;border-bottom:1px solid var(--border);font-size:.82rem;line-height:1.5}
.llm-entry .iter-label{font-weight:600;color:var(--accent);font-size:.75rem;margin-bottom:4px} .llm-entry .iter-label{font-weight:600;color:var(--accent);font-size:.75rem;margin-bottom:4px}
/* Config editor */
.config-section{margin-top:16px} .config-section{margin-top:16px}
.config-toggle{cursor:pointer;user-select:none;display:flex;align-items:center;gap:6px} .config-toggle{cursor:pointer;user-select:none;display:flex;align-items:center;gap:6px}
.config-toggle .arrow{transition:transform .2s;font-size:.7rem} .config-toggle .arrow{transition:transform .2s;font-size:.7rem}
@ -195,22 +177,19 @@ tr:hover{background:var(--card-hover)}
textarea.config-editor{width:100%;height:300px;background:var(--bg);color:var(--text);border:1px solid var(--border);border-radius:6px;padding:12px;font-family:var(--mono);font-size:.8rem;resize:vertical} textarea.config-editor{width:100%;height:300px;background:var(--bg);color:var(--text);border:1px solid var(--border);border-radius:6px;padding:12px;font-family:var(--mono);font-size:.8rem;resize:vertical}
.config-actions{display:flex;gap:8px;margin-top:8px} .config-actions{display:flex;gap:8px;margin-top:8px}
/* Downloads */
.downloads{display:flex;gap:8px;margin-top:16px;flex-wrap:wrap} .downloads{display:flex;gap:8px;margin-top:16px;flex-wrap:wrap}
.downloads a{color:var(--accent);text-decoration:none;font-size:.82rem;padding:6px 12px;border:1px solid var(--accent);border-radius:6px;transition:all .15s} .downloads a{color:var(--accent);text-decoration:none;font-size:.82rem;padding:6px 12px;border:1px solid var(--accent);border-radius:6px;transition:all .15s}
.downloads a:hover{background:var(--accent);color:#000} .downloads a:hover{background:var(--accent);color:#000}
/* Footer */
.footer{text-align:center;color:var(--text-dim);font-size:.75rem;padding:20px 0;margin-top:16px;border-top:1px solid var(--border)} .footer{text-align:center;color:var(--text-dim);font-size:.75rem;padding:20px 0;margin-top:16px;border-top:1px solid var(--border)}
</style> </style>
</head> </head>
<body> <body>
<div class="container"> <div class="container">
<!-- Header -->
<div class="header"> <div class="header">
<div> <div>
<h1><span class="btc">&#x20BF;</span> ML Strategy Optimizer</h1> <h1><span class="btc">&#x20BF;</span> Accumulation Signal Optimizer</h1>
<div style="margin-top:8px"> <div style="margin-top:8px">
<span id="statusBadge" class="status-badge status-idle"><span class="pulse"></span> Idle</span> <span id="statusBadge" class="status-badge status-idle"><span class="pulse"></span> Idle</span>
</div> </div>
@ -220,47 +199,41 @@ textarea.config-editor{width:100%;height:300px;background:var(--bg);color:var(--
<button id="btnStart" class="btn btn-start" onclick="startOpt()">Start Optimization</button> <button id="btnStart" class="btn btn-start" onclick="startOpt()">Start Optimization</button>
<button id="btnStop" class="btn btn-stop" onclick="stopOpt()" disabled>Stop</button> <button id="btnStop" class="btn btn-stop" onclick="stopOpt()" disabled>Stop</button>
</div> </div>
<div class="best-sharpe"> <div class="best-score">
<div class="label">Best Sharpe Ratio</div> <div class="label">Best Cost Improvement</div>
<div class="value" id="bestSharpe">0.000</div> <div class="value" id="bestScore">0.0<span class="unit">%</span></div>
</div> </div>
</div> </div>
</div> </div>
<!-- Main grid -->
<div class="grid"> <div class="grid">
<div class="left"> <div class="left">
<!-- Iteration Table -->
<div class="card" style="margin-bottom:16px"> <div class="card" style="margin-bottom:16px">
<h2>Iterations</h2> <h2>Iterations</h2>
<div class="table-wrap"> <div class="table-wrap">
<table> <table>
<thead> <thead>
<tr><th>#</th><th>Sharpe</th><th>Return%</th><th>MaxDD%</th><th>WinRate</th><th>Trades</th><th>PF</th><th>Model</th></tr> <tr><th>#</th><th>Cost Imp%</th><th>Signals</th><th>Frequency</th><th>R2</th><th>Bottoms</th><th>Tops</th><th>Model</th></tr>
</thead> </thead>
<tbody id="iterBody"></tbody> <tbody id="iterBody"></tbody>
</table> </table>
</div> </div>
</div> </div>
<!-- Equity Curve Chart -->
<div class="card"> <div class="card">
<h2>Performance Over Iterations</h2> <h2>Cost Improvement Over Iterations</h2>
<div class="chart-container"> <div class="chart-container">
<canvas id="sharpeChart"></canvas> <canvas id="mainChart"></canvas>
</div> </div>
</div> </div>
</div> </div>
<div class="right"> <div class="right">
<!-- LLM Analysis -->
<div class="card" style="margin-bottom:16px"> <div class="card" style="margin-bottom:16px">
<h2>LLM Analysis</h2> <h2>LLM Analysis</h2>
<div class="llm-panel" id="llmPanel"> <div class="llm-panel" id="llmPanel">
<div style="color:var(--text-dim);font-size:.82rem;padding:10px">No suggestions yet.</div> <div style="color:var(--text-dim);font-size:.82rem;padding:10px">No suggestions yet.</div>
</div> </div>
</div> </div>
<!-- Downloads -->
<div class="card"> <div class="card">
<h2>Downloads</h2> <h2>Downloads</h2>
<div class="downloads"> <div class="downloads">
@ -271,7 +244,6 @@ textarea.config-editor{width:100%;height:300px;background:var(--bg);color:var(--
</div> </div>
</div> </div>
<!-- Config Editor (collapsible) -->
<div class="card config-section"> <div class="card config-section">
<div class="config-toggle" id="configToggle" onclick="toggleConfig()"> <div class="config-toggle" id="configToggle" onclick="toggleConfig()">
<span class="arrow">&#9654;</span> <span class="arrow">&#9654;</span>
@ -286,22 +258,21 @@ textarea.config-editor{width:100%;height:300px;background:var(--bg);color:var(--
</div> </div>
</div> </div>
<div class="footer">BTC ML Trading Strategy Optimizer &mdash; VPS &rarr; Windows GPU &rarr; Mac Mini LLM</div> <div class="footer">BTC Accumulation Signal Optimizer &mdash; VPS &rarr; Windows GPU &rarr; Mac Mini LLM</div>
</div> </div>
<script> <script>
let chart = null; let chart = null;
let pollInterval = null; let pollInterval = null;
// Init chart
function initChart() { function initChart() {
const ctx = document.getElementById('sharpeChart').getContext('2d'); const ctx = document.getElementById('mainChart').getContext('2d');
chart = new Chart(ctx, { chart = new Chart(ctx, {
type: 'line', type: 'line',
data: { data: {
labels: [], labels: [],
datasets: [{ datasets: [{
label: 'Sharpe Ratio', label: 'Cost Improvement %',
data: [], data: [],
borderColor: '#f7931a', borderColor: '#f7931a',
backgroundColor: 'rgba(247,147,26,0.1)', backgroundColor: 'rgba(247,147,26,0.1)',
@ -311,7 +282,7 @@ function initChart() {
pointRadius: 4, pointRadius: 4,
pointBackgroundColor: '#f7931a' pointBackgroundColor: '#f7931a'
}, { }, {
label: 'Return %', label: 'Signal Count',
data: [], data: [],
borderColor: '#22c55e', borderColor: '#22c55e',
borderWidth: 1.5, borderWidth: 1.5,
@ -331,8 +302,8 @@ function initChart() {
}, },
scales: { scales: {
x: { ticks: { color: '#94a3b8' }, grid: { color: '#1e293b' } }, x: { ticks: { color: '#94a3b8' }, grid: { color: '#1e293b' } },
y: { position: 'left', ticks: { color: '#f7931a' }, grid: { color: '#1e293b' }, title: { display: true, text: 'Sharpe', color: '#f7931a' } }, y: { position: 'left', ticks: { color: '#f7931a' }, grid: { color: '#1e293b' }, title: { display: true, text: 'Cost Improvement %', color: '#f7931a' } },
y1: { position: 'right', ticks: { color: '#22c55e' }, grid: { drawOnChartArea: false }, title: { display: true, text: 'Return %', color: '#22c55e' } } y1: { position: 'right', ticks: { color: '#22c55e' }, grid: { drawOnChartArea: false }, title: { display: true, text: 'Signal Count', color: '#22c55e' } }
} }
} }
}); });
@ -353,7 +324,7 @@ function updateStatusBadge(status) {
document.getElementById('btnStart').disabled = (state === 'running'); document.getElementById('btnStart').disabled = (state === 'running');
document.getElementById('btnStop').disabled = (state !== 'running'); document.getElementById('btnStop').disabled = (state !== 'running');
document.getElementById('bestSharpe').textContent = (status.best_sharpe || 0).toFixed(3); document.getElementById('bestScore').innerHTML = (status.best_score || 0).toFixed(1) + '<span class="unit">%</span>';
} }
function updateIterations(iterations) { function updateIterations(iterations) {
@ -362,24 +333,23 @@ function updateIterations(iterations) {
tbody.innerHTML = '<tr><td colspan="8" style="color:var(--text-dim);text-align:center">No iterations yet</td></tr>'; tbody.innerHTML = '<tr><td colspan="8" style="color:var(--text-dim);text-align:center">No iterations yet</td></tr>';
return; return;
} }
const bestSharpe = Math.max(...iterations.map(i => i.sharpe || 0)); const bestCI = Math.max(...iterations.map(i => i.cost_improvement || 0));
let html = ''; let html = '';
for (const it of iterations) { for (const it of iterations) {
const isBest = it.sharpe === bestSharpe && bestSharpe > 0; const isBest = it.cost_improvement === bestCI && bestCI > 0;
const sc = it.sharpe > 1.5 ? 'var(--green)' : it.sharpe > 1.0 ? 'var(--yellow)' : 'var(--red)'; const sc = it.cost_improvement > 15 ? 'var(--green)' : it.cost_improvement > 10 ? 'var(--yellow)' : 'var(--red)';
html += '<tr class="' + (isBest ? 'best-row' : '') + '">'; html += '<tr class="' + (isBest ? 'best-row' : '') + '">';
html += '<td>' + it.iteration + '</td>'; html += '<td>' + it.iteration + '</td>';
html += '<td style="color:' + sc + ';font-weight:600">' + (it.sharpe||0).toFixed(3) + '</td>'; html += '<td style="color:' + sc + ';font-weight:600">' + (it.cost_improvement||0).toFixed(1) + '</td>';
html += '<td>' + (it["return"]||0).toFixed(1) + '</td>'; html += '<td>' + (it.signal_count||0) + '</td>';
html += '<td>' + (it.max_drawdown||0).toFixed(1) + '</td>'; html += '<td>' + (it.signal_frequency||0).toFixed(1) + '%</td>';
html += '<td>' + ((it.win_rate||0)*100).toFixed(1) + '%</td>'; html += '<td>' + (it.r2_score||0).toFixed(4) + '</td>';
html += '<td>' + (it.trades||0) + '</td>'; html += '<td>' + (it.score_at_bottoms||0).toFixed(1) + '</td>';
html += '<td>' + (it.profit_factor||0).toFixed(2) + '</td>'; html += '<td>' + (it.score_at_tops||0).toFixed(1) + '</td>';
html += '<td>' + (it.model_type||'—') + '</td>'; html += '<td>' + (it.model_type||'-') + '</td>';
html += '</tr>'; html += '</tr>';
} }
tbody.innerHTML = html; tbody.innerHTML = html;
// auto-scroll to bottom
const wrap = tbody.closest('.table-wrap'); const wrap = tbody.closest('.table-wrap');
wrap.scrollTop = wrap.scrollHeight; wrap.scrollTop = wrap.scrollHeight;
} }
@ -387,8 +357,8 @@ function updateIterations(iterations) {
function updateChart(iterations) { function updateChart(iterations) {
if (!chart || !iterations.length) return; if (!chart || !iterations.length) return;
chart.data.labels = iterations.map(i => '#' + i.iteration); chart.data.labels = iterations.map(i => '#' + i.iteration);
chart.data.datasets[0].data = iterations.map(i => i.sharpe || 0); chart.data.datasets[0].data = iterations.map(i => i.cost_improvement || 0);
chart.data.datasets[1].data = iterations.map(i => i["return"] || 0); chart.data.datasets[1].data = iterations.map(i => i.signal_count || 0);
chart.update('none'); chart.update('none');
} }
@ -468,14 +438,9 @@ async function updateConfig() {
async function resetConfig() { async function resetConfig() {
if (!confirm('Reset to initial config?')) return; if (!confirm('Reset to initial config?')) return;
try { try { location.reload(); } catch(e) { alert(e); }
const r = await fetch('/api/config');
// Fetch the initial config by reading it — for now just reload
location.reload();
} catch(e) { alert(e); }
} }
// Init
initChart(); initChart();
poll(); poll();
pollInterval = setInterval(poll, 10000); pollInterval = setInterval(poll, 10000);

213
llm_client/analyzer.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
LLM Strategy Analyzer -- Calls Ollama on Mac Mini to analyze results LLM Accumulation Signal Analyzer -- Calls Ollama on Mac Mini to analyze results
and suggest config modifications for the next iteration. and suggest config modifications for the next iteration.
""" """
@ -11,121 +11,130 @@ import requests
OLLAMA_URL = "http://100.100.242.21:11434" OLLAMA_URL = "http://100.100.242.21:11434"
MODEL = "qwen3.5:27b" MODEL = "qwen3.5:27b"
SYSTEM_PROMPT = """You are a quantitative trading strategy optimizer. You analyze ML model backtesting results for a BTC/USDT trading strategy and suggest precise modifications to improve performance. SYSTEM_PROMPT = """You are a quantitative analyst optimizing a BTC ACCUMULATION SIGNAL model. The goal is NOT day-trading -- it is finding statistically optimal times to BUY BTC for long-term holding.
## Your Task ## Core Question
Given the current configuration and results, suggest 1-3 specific, justified changes to the configuration for the next iteration. Be methodical and scientific -- change one thing at a time when possible. "Given current market conditions, is NOW a good time to BUY BTC for long-term holding?"
## What the Model Does
For each candle, the model predicts an Accumulation Score (0-100):
- 90-100: STRONG BUY -- historically rare, excellent entry point
- 70-89: GOOD BUY -- better than average entry
- 50-69: NEUTRAL -- average time to buy
- 30-49: WAIT -- price likely to come down
- 0-29: POOR -- historically bad time to buy (near local tops)
The model is trained on ACTUAL forward returns at 7d, 30d, and 90d horizons, weighted 20/30/50. Times when buying led to the best long-term returns get the highest scores.
## Primary Metric: cost_basis_improvement_pct
This measures how much better the model's average buy price is vs uniform DCA.
- 10%+ = good
- 15%+ = excellent
- 20%+ = exceptional
Also require strong_buy_signal_count >= 30 for statistical validity.
## Config Parameters You Can Modify ## Config Parameters You Can Modify
**model_type**: "xgboost", "lightgbm", "catboost", "ensemble", "lstm", or "hybrid" **model_type**: "xgboost", "lightgbm", "catboost", "lstm", or "hybrid"
- xgboost: Generally best for structured data, fast GPU training - hybrid: Average of LSTM + XGBoost regression predictions. Recommended default.
- lightgbm: Faster training, good with large feature sets - xgboost: Fast GPU training, good for structured features.
- catboost: Handles feature interactions well, less tuning needed - lstm: Captures temporal patterns in price sequences.
- ensemble: Combines xgboost+lightgbm+catboost, reduces variance but slower
- lstm: PyTorch LSTM neural network, captures temporal/sequential patterns in price data
- hybrid: Combines LSTM (60% weight) + XGBoost (40% weight). Only enters trades when BOTH models agree on direction. The hybrid model typically outperforms single models -- LSTM captures temporal patterns while XGBoost handles feature interactions. Recommended as default.
**hyperparameters** (gradient boosting): **hyperparameters** (gradient boosting):
- learning_rate (0.001-0.3): Lower = more robust but slower. If overfitting, decrease. - learning_rate (0.001-0.1): Lower = more robust. Start conservative.
- max_depth (3-10): Controls model complexity. Deeper = more overfitting risk. - max_depth (3-8): Controls complexity. Deeper risks overfitting.
- n_estimators (100-2000): More trees = better fit but diminishing returns. - n_estimators (200-1500): More trees = better fit but diminishing returns.
- subsample (0.5-1.0): Row sampling. Lower = more regularization. - subsample (0.5-1.0): Row sampling for regularization.
- colsample_bytree (0.5-1.0): Feature sampling per tree. Lower = more diversity. - colsample_bytree (0.5-1.0): Feature sampling per tree.
- min_child_weight (1-20): Higher = more conservative splits. - min_child_weight (5-30): Higher = more conservative (important for noisy targets).
- gamma (0-5): Minimum loss reduction for split. Higher = more pruning. - gamma (0-5): Minimum loss reduction for split.
- reg_alpha (0-10): L1 regularization. Encourages sparsity. - reg_alpha (0-10): L1 regularization.
- reg_lambda (0-10): L2 regularization. Prevents large weights. - reg_lambda (1-10): L2 regularization. Higher values prevent overfitting.
**hyperparameters** (LSTM-specific, used by lstm and hybrid model_types): **hyperparameters** (LSTM):
- lstm_hidden_size (32-256): LSTM hidden units. Larger = more capacity but overfitting risk. Default 128. - lstm_hidden_size (32-256): Hidden units.
- lstm_num_layers (1-4): Stacked LSTM layers. 2 is usually optimal. More layers need more data. - lstm_num_layers (1-4): Stacked layers. 2 is usually optimal.
- lstm_dropout (0.1-0.5): Dropout between LSTM layers and before output. Higher = more regularization. - lstm_dropout (0.1-0.5): Regularization.
- lstm_epochs (50-200): Max training epochs. Early stopping usually triggers before this. - lstm_epochs (50-200): Max training epochs (early stopping usually triggers).
- lstm_batch_size (32-128): Training batch size. Smaller = noisier gradients but better generalization. - lstm_batch_size (32-128): Smaller = noisier but better generalization.
- lstm_sequence_length (10-50): How many past candles the LSTM sees per prediction. Longer = more context but more memory. Default 20. - lstm_sequence_length (15-60): Past candles the LSTM sees. Longer = more context.
- lstm_patience (5-20): Early stopping patience on validation loss. Lower = stop sooner. - lstm_patience (5-20): Early stopping patience.
**target**: **target**:
- direction: "long", "short", or "both" - forward_periods_4h: List of 3 forward periods in 4h candles [short, medium, long].
- horizon_candles (1-20): How far ahead to predict. Longer = smoother but lagging. Defaults: [42, 180, 540] = roughly [7d, 30d, 90d]
- threshold_pct (0.3-3.0): Minimum move % to label as positive. Higher = fewer but clearer signals. - weights: Weights for each period. Default [0.2, 0.3, 0.5] (emphasize long-term).
- score_range: [0, 100] -- do not change.
**strategy**: **strategy**:
- entry_threshold (0.5-0.8): Min prediction probability to enter trade. Higher = fewer trades, higher quality. - strong_buy_threshold (70-95): Score above which = STRONG BUY signal. Higher = fewer but better signals.
- stop_loss_pct (0.5-5.0): Max loss before exit (used when dynamic_sl_tp is false). - good_buy_threshold (50-80): Score above which = GOOD BUY. Used for cost basis comparison.
- take_profit_pct (1.0-10.0): Target profit (used when dynamic_sl_tp is false). Should be > stop_loss for positive expectancy. - poor_threshold (10-40): Score below which = POOR time to buy.
- trailing_stop_pct (0.5-3.0): Trailing stop distance. Tighter = locks profit faster but exits early.
- min_confidence_to_trade (0.5-0.9): Absolute minimum confidence to consider.
- exit_type: "trailing_stop" or "fixed" (just SL/TP)
- dynamic_sl_tp (true/false): Use ATR-based dynamic stop-loss and take-profit instead of fixed percentages. Adapts to current volatility. Recommended: true.
- atr_sl_multiplier (1.0-3.0): ATR multiplier for stop-loss. E.g., 1.5 means SL = 1.5 * ATR(14). Lower = tighter stops.
- atr_tp_multiplier (2.0-5.0): ATR multiplier for take-profit. E.g., 3.0 means TP = 3.0 * ATR(14). Should be > atr_sl_multiplier.
**features**: **features**:
- use_volume_features (true/false): Volume features can be noisy in crypto. - use_price_position (true/false): Distance from ATH, 52w high/low, percentile.
- use_candle_patterns (true/false): Candle patterns may or may not help. - use_momentum (true/false): RSI, MACD, Stochastic, Williams %R, ROC.
- use_lag_features (true/false): Lagged features capture momentum. - use_volatility (true/false): Bollinger Bands, ATR, consecutive red candles, drawdown.
- lag_periods: List of lag periods [1,2,3,5,10] - use_volume (true/false): Volume ratio, OBV, red/green volume ratio.
- lookback_periods: List of lookback windows [3,5,10,20] - use_cycle (true/false): MA cross regime, candles since major drawdown.
- use_scaler (true/false): Apply StandardScaler normalization to all features. Critical for LSTM, also helps gradient boosting. Recommended: true. - use_pca (true/false): PCA dimensionality reduction.
- use_pca (true/false): Apply PCA dimensionality reduction after scaling. Reduces noise and multicollinearity. Recommended with many features. - pca_variance (0.80-0.99): Variance to retain.
- pca_variance (0.80-0.99): Fraction of variance to retain with PCA. 0.95 keeps 95% of information. Lower = fewer dimensions, more noise removed. - use_scaler (true/false): StandardScaler. Critical for LSTM.
**training**: **training**:
- walk_forward_windows (3-10): More windows = more robust but less data per window. Used when rolling_window is false. - rolling_window (true/false): Rolling vs static walk-forward.
- rolling_window (true/false): Use rolling window instead of static walk-forward splits. Trains on last N candles, tests on next M, slides forward. More realistic for time series. Recommended: true. - rolling_train_size (1500-5000): Training window candles.
- rolling_train_size (1000-5000): Number of candles in the rolling training window. Larger = more data but older patterns. - rolling_test_size (100-500): Test window candles.
- rolling_test_size (100-500): Number of candles in the rolling test window. Smaller = more retraining, better adaptation.
## Key Metrics to Optimize (in priority order) ## Key Metrics to Analyze
1. **Sharpe Ratio** (target: > 2.0): Risk-adjusted return. Most important metric. 1. **cost_basis_improvement_pct**: PRIMARY metric. How much better is model buy price vs DCA.
2. **Profit Factor** (target: > 1.5): Gross profit / gross loss. 2. **strong_buy_signal_count**: Must be >= 30 for validity. Too few = raise threshold. Too many = lower it.
3. **Max Drawdown** (target: > -15%): Worst peak-to-trough decline. 3. **signal_frequency_pct**: Should be 5-15%. If outside, adjust thresholds.
4. **Win Rate** (target: > 55%): Percentage of winning trades. 4. **avg_score_at_actual_bottoms**: Should be high (>70). Model should recognize bottoms.
5. **Trade Count**: Need enough trades for statistical significance (>50). 5. **avg_score_at_actual_tops**: Should be low (<30). Model should avoid tops.
6. **model_r2_score**: Regression fit quality. > 0.2 is decent for financial data.
7. **per_window_cost_improvement**: Consistency across windows. Low variance = robust.
## Decision Guidelines ## Decision Guidelines
- If Sharpe < 1.0: The strategy is not working well. Consider larger changes (switch to hybrid, enable PCA/scaler, adjust target). - If cost_improvement < 5%: Strategy is barely working. Try: switch model type, enable all features, increase training window, lower good_buy_threshold.
- If Sharpe 1.0-1.5: Decent. Fine-tune hyperparameters and thresholds. - If cost_improvement 5-10%: Decent. Fine-tune thresholds and hyperparameters.
- If Sharpe 1.5-2.0: Good. Make small, targeted improvements. - If cost_improvement 10-15%: Good. Make targeted improvements -- focus on signal consistency.
- If Sharpe > 2.0: Very good. Be careful not to overfit. - If cost_improvement > 15%: Very good. Be careful not to overfit. Check per_window variance.
- If win_rate < 0.50 but profit_factor > 1.5: Strategy relies on big wins -- ok, tighten SL. - If signal_count < 30: Not statistically valid. Lower strong_buy_threshold, increase training data.
- If win_rate > 0.60 but profit_factor < 1.2: Many small wins but losses are too big -- widen TP or tighten SL. - If signal_frequency > 20%: Too many signals = not selective enough. Raise threshold.
- If trade_count < 30: Not enough trades. Lower entry_threshold or min_confidence. - If signal_frequency < 3%: Too few signals. Lower threshold.
- If max_drawdown < -20%: Too risky. Increase regularization, tighten stop loss, enable dynamic_sl_tp. - If score_at_bottoms < 60: Model is missing bottoms. More features, different model type.
- If per_window_sharpe has high variance: Model is not stable. More regularization, enable PCA, or try hybrid. - If score_at_tops > 40: Model is not avoiding tops. More regularization.
- Check feature_importances: If top features make financial sense, good. If random features dominate, possible overfitting -- enable PCA or reduce features. - If per_window has high variance: Model is unstable. Increase regularization, try hybrid.
- For LSTM/hybrid: if underfitting, increase lstm_hidden_size or lstm_num_layers. If overfitting, increase lstm_dropout or decrease lstm_sequence_length. - Check feature_importances: price position features should dominate (distance from ATH, percentile).
- The hybrid model combining LSTM + XGBoost typically outperforms single models. LSTM captures temporal patterns while XGBoost handles feature interactions. Use hybrid as the default unless you have a specific reason not to.
## Response Format ## Response Format
You MUST respond with ONLY a JSON object (no markdown, no explanation outside the JSON): You MUST respond with ONLY a JSON object (no markdown, no explanation outside the JSON):
``` ```
{ {
"reasoning": "Explanation of what you observed and why you're making these changes", "reasoning": "Explanation of observations and why you are making these changes",
"changes": ["Change 1 description", "Change 2 description"], "changes": ["Change 1 description", "Change 2 description"],
"config": { <complete modified config JSON> } "config": { <complete modified config JSON> }
} }
``` ```
The "config" field must contain the COMPLETE config (not just changes) so it can be used directly.""" The "config" field must contain the COMPLETE config so it can be used directly."""
def analyze_and_suggest(current_config: dict, results: dict, def analyze_and_suggest(current_config, results, iteration_history=None):
iteration_history: list = None) -> tuple[dict, str]:
""" """
Send current results to LLM and get suggested config modifications. Send current results to LLM and get suggested config modifications.
Returns (new_config, reasoning). Returns (new_config, reasoning).
""" """
# Build the user prompt with context
history_text = "" history_text = ""
if iteration_history: if iteration_history:
history_text = "\n## Previous Iterations (most recent last)\n" history_text = "\n## Previous Iterations (most recent last)\n"
for h in iteration_history[-5:]: for h in iteration_history[-5:]:
history_text += ( history_text += (
f"- Iteration {h['iteration']}: Sharpe={h['sharpe']}, " f"- Iteration {h.get('iteration', '?')}: "
f"Return={h['return']}%, WinRate={h['win_rate']}, " f"CostImprovement={h.get('cost_improvement', 0):.1f}%, "
f"Trades={h['trades']}, Model={h['model_type']}\n" f"Signals={h.get('signal_count', 0)}, "
f"R2={h.get('r2_score', 0):.4f}, "
f"Model={h.get('model_type', '?')}\n"
) )
user_prompt = f"""## Current Configuration user_prompt = f"""## Current Configuration
@ -134,21 +143,24 @@ def analyze_and_suggest(current_config: dict, results: dict,
``` ```
## Current Results ## Current Results
- Sharpe Ratio: {results.get('sharpe_ratio', 0)} - Cost Basis Improvement: {results.get('cost_basis_improvement_pct', 0):.1f}%
- Total Return: {results.get('total_return_pct', 0)}% - Avg Cost (Model): ${results.get('avg_cost_basis_model', 0):,.2f}
- Max Drawdown: {results.get('max_drawdown_pct', 0)}% - Avg Cost (DCA): ${results.get('avg_cost_basis_dca', 0):,.2f}
- Win Rate: {results.get('win_rate', 0)} - Strong Buy Signals: {results.get('strong_buy_signal_count', 0)}
- Trade Count: {results.get('trade_count', 0)} - Good Buy Signals: {results.get('good_buy_signal_count', 0)}
- Profit Factor: {results.get('profit_factor', 0)} - Signal Frequency: {results.get('signal_frequency_pct', 0):.1f}%
- Avg Trade Duration: {results.get('avg_trade_duration_candles', 0)} candles - Quality of Strong Buys: {results.get('pct_quality_strong_buy', 0):.1%}
- Per-Window Sharpe: {results.get('per_window_sharpe', [])} - Model R2: {results.get('model_r2_score', 0):.4f}
- Score at Actual Bottoms: {results.get('avg_score_at_actual_bottoms', 0):.1f}
- Score at Actual Tops: {results.get('avg_score_at_actual_tops', 0):.1f}
- Per-Window Improvement: {results.get('per_window_cost_improvement', [])}
- Score Distribution: {results.get('score_distribution', {})}
## Top Feature Importances ## Top Feature Importances
{json.dumps(dict(list(results.get('feature_importances', {}).items())[:15]), indent=2)} {json.dumps(dict(list(results.get('feature_importances', {}).items())[:15]), indent=2)}
{history_text} {history_text}
Analyze these results and suggest 1-3 specific modifications to the config. Return ONLY valid JSON.""" Analyze these results and suggest 1-3 specific modifications to the config. Return ONLY valid JSON."""
# Call Ollama
payload = { payload = {
"model": MODEL, "model": MODEL,
"messages": [ "messages": [
@ -168,7 +180,6 @@ Analyze these results and suggest 1-3 specific modifications to the config. Retu
resp.raise_for_status() resp.raise_for_status()
content = resp.json()["message"]["content"] content = resp.json()["message"]["content"]
# Parse JSON from response (handle markdown code blocks)
# Strip thinking tags if present # Strip thinking tags if present
content = re.sub(r"<think>.*?</think>", "", content, flags=re.DOTALL).strip() content = re.sub(r"<think>.*?</think>", "", content, flags=re.DOTALL).strip()
@ -176,8 +187,6 @@ Analyze these results and suggest 1-3 specific modifications to the config. Retu
if json_match: if json_match:
parsed = json.loads(json_match.group(1)) parsed = json.loads(json_match.group(1))
else: else:
# Try parsing the whole response as JSON
# Find the outermost JSON object
brace_start = content.find("{") brace_start = content.find("{")
if brace_start >= 0: if brace_start >= 0:
depth = 0 depth = 0
@ -198,7 +207,6 @@ Analyze these results and suggest 1-3 specific modifications to the config. Retu
changes = parsed.get("changes", []) changes = parsed.get("changes", [])
new_config = parsed.get("config", current_config) new_config = parsed.get("config", current_config)
# Validate that config has required fields
required_keys = ["model_type", "features", "target", "hyperparameters", "strategy", "training"] required_keys = ["model_type", "features", "target", "hyperparameters", "strategy", "training"]
for key in required_keys: for key in required_keys:
if key not in new_config: if key not in new_config:
@ -209,22 +217,25 @@ Analyze these results and suggest 1-3 specific modifications to the config. Retu
if __name__ == "__main__": if __name__ == "__main__":
# Test with dummy data
import sys import sys
config_path = sys.argv[1] if len(sys.argv) > 1 else "config/initial_config.json" config_path = sys.argv[1] if len(sys.argv) > 1 else "config/initial_config.json"
with open(config_path) as f: with open(config_path) as f:
config = json.load(f) config = json.load(f)
dummy_results = { dummy_results = {
"sharpe_ratio": 1.2, "cost_basis_improvement_pct": 8.5,
"total_return_pct": 15.3, "avg_cost_basis_model": 65000,
"max_drawdown_pct": -12.5, "avg_cost_basis_dca": 71000,
"win_rate": 0.55, "strong_buy_signal_count": 45,
"trade_count": 120, "good_buy_signal_count": 120,
"profit_factor": 1.4, "signal_frequency_pct": 7.2,
"avg_trade_duration_candles": 7.2, "pct_quality_strong_buy": 0.72,
"feature_importances": {"RSI_14": 0.15, "MACD_hist": 0.12, "BB_width": 0.10}, "model_r2_score": 0.22,
"per_window_sharpe": [1.0, 1.3, 1.5, 0.9, 1.1], "avg_score_at_actual_bottoms": 68.5,
"avg_score_at_actual_tops": 35.2,
"per_window_cost_improvement": [7.1, 9.3, 8.8, 10.2, 7.0],
"score_distribution": {"0-20": 80, "20-40": 150, "40-60": 200, "60-80": 130, "80-100": 40},
"feature_importances": {"dist_from_ath_pct": 0.18, "RSI_14": 0.12, "price_percentile_365": 0.10},
} }
new_config, reasoning = analyze_and_suggest(config, dummy_results) new_config, reasoning = analyze_and_suggest(config, dummy_results)

1037
ml_engine/train_and_backtest.py
View File

File diff suppressed because it is too large Load Diff

180
orchestrator.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
BTC ML Trading Strategy Optimizer — Orchestrator BTC Accumulation Signal Optimizer -- Orchestrator
Coordinates the optimization loop across VPS, Windows PC (GPU), and Mac Mini (LLM). Coordinates the optimization loop across VPS, Windows PC (GPU), and Mac Mini (LLM).
""" """
@ -28,7 +28,8 @@ MAC_MINI_HOST = "bizzle@bizzles-mac-mini-1"
MAX_ITERATIONS = 50 MAX_ITERATIONS = 50
CONVERGENCE_WINDOW = 5 CONVERGENCE_WINDOW = 5
CONVERGENCE_THRESHOLD = 0.01 # 1% improvement CONVERGENCE_THRESHOLD = 0.01 # 1% improvement
TARGET_SHARPE = 3.0 TARGET_COST_IMPROVEMENT = 20.0 # 20% cost basis improvement = exceptional
MIN_SIGNAL_COUNT = 30 # Minimum strong buy signals for valid results
ML_TIMEOUT = 600 # 10 minutes ML_TIMEOUT = 600 # 10 minutes
# Colors # Colors
@ -98,7 +99,6 @@ def run_ml_training():
) )
if result.returncode != 0: if result.returncode != 0:
raise RuntimeError(f"ML training failed:\n{result.stderr}\n{result.stdout}") raise RuntimeError(f"ML training failed:\n{result.stderr}\n{result.stdout}")
# Print training output
for line in result.stdout.strip().split("\n"): for line in result.stdout.strip().split("\n"):
log(f" {C.DIM}{line}", C.DIM) log(f" {C.DIM}{line}", C.DIM)
return True return True
@ -127,45 +127,53 @@ def check_convergence(history):
if len(history) < CONVERGENCE_WINDOW + 1: if len(history) < CONVERGENCE_WINDOW + 1:
return False, "Not enough iterations" return False, "Not enough iterations"
recent = history[-CONVERGENCE_WINDOW:] # Only consider valid results (enough signals)
sharpes = [h["sharpe"] for h in recent] valid = [h for h in history if h.get("signal_count", 0) >= MIN_SIGNAL_COUNT]
# Check if best sharpe exceeds target if not valid:
best_sharpe = max(h["sharpe"] for h in history) return False, "No valid results yet"
if best_sharpe >= TARGET_SHARPE:
return True, f"Target Sharpe reached: {best_sharpe:.3f}" recent = history[-CONVERGENCE_WINDOW:]
scores = [h.get("cost_improvement", 0) for h in recent]
# Check if best score exceeds target
best_score = max(h.get("cost_improvement", 0) for h in valid)
if best_score >= TARGET_COST_IMPROVEMENT:
return True, f"Target cost improvement reached: {best_score:.1f}%"
# Check if improvement has stalled # Check if improvement has stalled
best_recent = max(sharpes) best_recent = max(scores)
worst_recent = min(sharpes) worst_recent = min(scores)
if best_recent > 0 and (best_recent - worst_recent) / best_recent < CONVERGENCE_THRESHOLD: if best_recent > 0 and (best_recent - worst_recent) / best_recent < CONVERGENCE_THRESHOLD:
return True, f"Converged: Sharpe variance < {CONVERGENCE_THRESHOLD*100}% over {CONVERGENCE_WINDOW} iterations" return True, f"Converged: variance < {CONVERGENCE_THRESHOLD*100}% over {CONVERGENCE_WINDOW} iterations"
return False, "" return False, ""
def print_header(): def print_header():
print(f""" print(f"""
{C.BOLD}{C.CYAN}╔══════════════════════════════════════════════════╗ {C.BOLD}{C.CYAN}========================================================
║ BTC ML Trading Strategy Optimizer ║ BTC Accumulation Signal Optimizer
║ VPS → Windows GPU → Mac Mini LLM → Loop ║ VPS -> Windows GPU -> Mac Mini LLM -> Loop
╚══════════════════════════════════════════════════╝{C.RESET} ========================================================{C.RESET}
""") """)
def print_results(results, iteration): def print_results(results, iteration):
sharpe = results.get("sharpe_ratio", 0) cost_imp = results.get("cost_basis_improvement_pct", 0)
sharpe_color = C.GREEN if sharpe > 1.5 else C.YELLOW if sharpe > 1.0 else C.RED color = C.GREEN if cost_imp > 15 else C.YELLOW if cost_imp > 10 else C.RED
print(f""" print(f"""
{C.BOLD}━━━ Iteration {iteration} Results ━━━{C.RESET} {C.BOLD}--- Iteration {iteration} Results ---{C.RESET}
Sharpe Ratio: {sharpe_color}{C.BOLD}{sharpe:.3f}{C.RESET} Cost Improvement: {color}{C.BOLD}{cost_imp:.1f}%{C.RESET}
Total Return: {results.get('total_return_pct', 0):.1f}% Avg Cost (Model): ${results.get('avg_cost_basis_model', 0):,.2f}
Max Drawdown: {results.get('max_drawdown_pct', 0):.1f}% Avg Cost (DCA): ${results.get('avg_cost_basis_dca', 0):,.2f}
Win Rate: {results.get('win_rate', 0):.1%} Strong Signals: {results.get('strong_buy_signal_count', 0)}
Trade Count: {results.get('trade_count', 0)} Signal Frequency: {results.get('signal_frequency_pct', 0):.1f}%
Profit Factor: {results.get('profit_factor', 0):.3f} Quality Score: {results.get('pct_quality_strong_buy', 0):.1%}
Avg Duration: {results.get('avg_trade_duration_candles', 0):.1f} candles Model R2: {results.get('model_r2_score', 0):.4f}
Window Sharpes: {results.get('per_window_sharpe', [])} Score@Bottoms: {results.get('avg_score_at_actual_bottoms', 0):.1f}
Score@Tops: {results.get('avg_score_at_actual_tops', 0):.1f}
Window Improvements: {results.get('per_window_cost_improvement', [])}
""") """)
@ -173,14 +181,11 @@ def main():
print_header() print_header()
os.makedirs(RESULTS_DIR, exist_ok=True) os.makedirs(RESULTS_DIR, exist_ok=True)
# Step 1: Ensure data
ensure_data() ensure_data()
# Step 2: Load or create initial config
config_path = os.path.join(CONFIG_DIR, "initial_config.json") config_path = os.path.join(CONFIG_DIR, "initial_config.json")
best_config_path = os.path.join(CONFIG_DIR, "best_config.json") best_config_path = os.path.join(CONFIG_DIR, "best_config.json")
# Resume from best config if it exists
if os.path.exists(best_config_path): if os.path.exists(best_config_path):
log("Resuming from best_config.json", C.GREEN) log("Resuming from best_config.json", C.GREEN)
with open(best_config_path) as f: with open(best_config_path) as f:
@ -191,29 +196,24 @@ def main():
history = load_iteration_history() history = load_iteration_history()
start_iter = len(history) + 1 start_iter = len(history) + 1
best_sharpe = max((h["sharpe"] for h in history), default=0) best_score = max((h.get("cost_improvement", 0) for h in history), default=0)
log(f"Starting at iteration {start_iter}, best Sharpe so far: {best_sharpe:.3f}", C.BOLD) log(f"Starting at iteration {start_iter}, best cost improvement so far: {best_score:.1f}%", C.BOLD)
# Step 3: Setup Windows remote
setup_windows_remote() setup_windows_remote()
# SCP the ML engine script (once)
log("Uploading ML engine to Windows...", C.CYAN) log("Uploading ML engine to Windows...", C.CYAN)
scp_to_windows(os.path.join(BASE_DIR, "ml_engine", "train_and_backtest.py"), "train_and_backtest.py") scp_to_windows(os.path.join(BASE_DIR, "ml_engine", "train_and_backtest.py"), "train_and_backtest.py")
# SCP data files (once)
for tf in ["1h", "4h"]: for tf in ["1h", "4h"]:
data_file = os.path.join(DATA_DIR, f"btc_{tf}.csv") data_file = os.path.join(DATA_DIR, f"btc_{tf}.csv")
if os.path.exists(data_file): if os.path.exists(data_file):
log(f"Uploading btc_{tf}.csv to Windows...", C.CYAN) log(f"Uploading btc_{tf}.csv to Windows...", C.CYAN)
scp_to_windows(data_file, f"btc_{tf}.csv") scp_to_windows(data_file, f"btc_{tf}.csv")
# Import LLM analyzer
sys.path.insert(0, os.path.join(BASE_DIR, "llm_client")) sys.path.insert(0, os.path.join(BASE_DIR, "llm_client"))
from analyzer import analyze_and_suggest from analyzer import analyze_and_suggest
# Main optimization loop
for iteration in range(start_iter, MAX_ITERATIONS + 1): for iteration in range(start_iter, MAX_ITERATIONS + 1):
log(f"\n{'='*50}", C.BOLD) log(f"\n{'='*50}", C.BOLD)
log(f"ITERATION {iteration}/{MAX_ITERATIONS}", f"{C.BOLD}{C.CYAN}") log(f"ITERATION {iteration}/{MAX_ITERATIONS}", f"{C.BOLD}{C.CYAN}")
@ -222,13 +222,11 @@ def main():
f"Depth: {config.get('hyperparameters', {}).get('max_depth', '?')}", C.DIM) f"Depth: {config.get('hyperparameters', {}).get('max_depth', '?')}", C.DIM)
log(f"{'='*50}", C.BOLD) log(f"{'='*50}", C.BOLD)
# Write current config to temp file and SCP
tmp_config = os.path.join(BASE_DIR, "config", "current_config.json") tmp_config = os.path.join(BASE_DIR, "config", "current_config.json")
with open(tmp_config, "w") as f: with open(tmp_config, "w") as f:
json.dump(config, f, indent=2) json.dump(config, f, indent=2)
scp_to_windows(tmp_config, "config.json") scp_to_windows(tmp_config, "config.json")
# Run ML training on Windows
try: try:
run_ml_training() run_ml_training()
except (RuntimeError, subprocess.TimeoutExpired) as e: except (RuntimeError, subprocess.TimeoutExpired) as e:
@ -238,7 +236,6 @@ def main():
config = history[-1].get("config", config) config = history[-1].get("config", config)
continue continue
# Fetch results from Windows
results_local = os.path.join(RESULTS_DIR, f"results_iter_{iteration}.json") results_local = os.path.join(RESULTS_DIR, f"results_iter_{iteration}.json")
scp_from_windows("results.json", results_local) scp_from_windows("results.json", results_local)
@ -247,34 +244,35 @@ def main():
print_results(results, iteration) print_results(results, iteration)
# Track best current_score = results.get("cost_basis_improvement_pct", 0)
current_sharpe = results.get("sharpe_ratio", 0) signal_count = results.get("strong_buy_signal_count", 0)
is_best = current_sharpe > best_sharpe is_best = current_score > best_score and signal_count >= MIN_SIGNAL_COUNT
if is_best: if is_best:
best_sharpe = current_sharpe best_score = current_score
with open(best_config_path, "w") as f: with open(best_config_path, "w") as f:
json.dump(config, f, indent=2) json.dump(config, f, indent=2)
log(f"NEW BEST! Sharpe: {best_sharpe:.3f}", f"{C.BOLD}{C.GREEN}") log(f"NEW BEST! Cost Improvement: {best_score:.1f}%", f"{C.BOLD}{C.GREEN}")
# Log iteration
iter_data = { iter_data = {
"iteration": iteration, "iteration": iteration,
"timestamp": datetime.now(timezone.utc).isoformat(), "timestamp": datetime.now(timezone.utc).isoformat(),
"sharpe": current_sharpe, "cost_improvement": current_score,
"return": results.get("total_return_pct", 0), "avg_30d_return": results.get("avg_quality_score_strong_buy", 0),
"max_drawdown": results.get("max_drawdown_pct", 0), "avg_90d_return": results.get("pct_quality_strong_buy", 0),
"win_rate": results.get("win_rate", 0), "signal_count": signal_count,
"trades": results.get("trade_count", 0), "signal_frequency": results.get("signal_frequency_pct", 0),
"profit_factor": results.get("profit_factor", 0), "r2_score": results.get("model_r2_score", 0),
"score_at_bottoms": results.get("avg_score_at_actual_bottoms", 0),
"score_at_tops": results.get("avg_score_at_actual_tops", 0),
"model_type": config.get("model_type", "unknown"), "model_type": config.get("model_type", "unknown"),
"is_best": is_best, "is_best": is_best,
"config": config, "config": config,
"results": results,
} }
save_iteration(iter_data) save_iteration(iter_data)
history.append(iter_data) history.append(iter_data)
# Check convergence
converged, reason = check_convergence(history) converged, reason = check_convergence(history)
if converged: if converged:
log(f"\nOptimization converged: {reason}", f"{C.BOLD}{C.GREEN}") log(f"\nOptimization converged: {reason}", f"{C.BOLD}{C.GREEN}")
@ -284,17 +282,15 @@ def main():
log(f"\nMax iterations ({MAX_ITERATIONS}) reached.", C.YELLOW) log(f"\nMax iterations ({MAX_ITERATIONS}) reached.", C.YELLOW)
break break
# Ask LLM for next config
log("\nConsulting LLM for strategy modifications...", C.MAGENTA) log("\nConsulting LLM for strategy modifications...", C.MAGENTA)
try: try:
summary_history = [ summary_history = [
{ {
"iteration": h["iteration"], "iteration": h["iteration"],
"sharpe": h["sharpe"], "cost_improvement": h.get("cost_improvement", 0),
"return": h["return"], "signal_count": h.get("signal_count", 0),
"win_rate": h["win_rate"], "r2_score": h.get("r2_score", 0),
"trades": h["trades"], "model_type": h.get("model_type", "unknown"),
"model_type": h["model_type"],
} }
for h in history for h in history
] ]
@ -304,21 +300,19 @@ def main():
except Exception as e: except Exception as e:
log(f"LLM call failed: {e}", C.RED) log(f"LLM call failed: {e}", C.RED)
log("Continuing with current config + random perturbation...", C.YELLOW) log("Continuing with current config + random perturbation...", C.YELLOW)
# Small random perturbation as fallback
import random import random
hp = config.get("hyperparameters", {}) hp = config.get("hyperparameters", {})
hp["learning_rate"] = hp.get("learning_rate", 0.05) * random.uniform(0.8, 1.2) hp["learning_rate"] = hp.get("learning_rate", 0.01) * random.uniform(0.8, 1.2)
hp["max_depth"] = max(3, min(10, hp.get("max_depth", 6) + random.choice([-1, 0, 1]))) hp["max_depth"] = max(3, min(10, hp.get("max_depth", 5) + random.choice([-1, 0, 1])))
config["hyperparameters"] = hp config["hyperparameters"] = hp
# Final summary
print(f""" print(f"""
{C.BOLD}{C.GREEN}╔══════════════════════════════════════════════════╗ {C.BOLD}{C.GREEN}========================================================
║ Optimization Complete! ║ Optimization Complete!
╚══════════════════════════════════════════════════╝{C.RESET} ========================================================{C.RESET}
Total Iterations: {len(history)} Total Iterations: {len(history)}
Best Sharpe: {C.BOLD}{best_sharpe:.3f}{C.RESET} Best Cost Improvement: {C.BOLD}{best_score:.1f}%{C.RESET}
Best Config: {best_config_path} Best Config: {best_config_path}
Iteration Log: {ITERATIONS_LOG} Iteration Log: {ITERATIONS_LOG}
""") """)
@ -326,15 +320,14 @@ def main():
# --- Library API for dashboard integration --- # --- Library API for dashboard integration ---
# Shared state for dashboard
_stop_event = threading.Event() _stop_event = threading.Event()
_status = { _status = {
"state": "idle", # idle, running, completed, error "state": "idle",
"iteration": 0, "iteration": 0,
"max_iterations": MAX_ITERATIONS, "max_iterations": MAX_ITERATIONS,
"best_sharpe": 0.0, "best_score": 0.0,
"error": None, "error": None,
"llm_suggestions": [], # list of {iteration, reasoning, changes} "llm_suggestions": [],
} }
_status_lock = threading.Lock() _status_lock = threading.Lock()
@ -352,15 +345,9 @@ def update_status(**kwargs):
def run_optimization_loop(callback=None, config_override=None): def run_optimization_loop(callback=None, config_override=None):
""" """Run the optimization loop from a background thread."""
Run the optimization loop. Designed to be called from a background thread.
Args:
callback: Called after each iteration with (iteration_number, iter_data_dict).
config_override: Optional dict to use instead of loading from disk.
"""
_stop_event.clear() _stop_event.clear()
update_status(state="running", iteration=0, error=None, best_sharpe=0.0) update_status(state="running", iteration=0, error=None, best_score=0.0)
try: try:
os.makedirs(RESULTS_DIR, exist_ok=True) os.makedirs(RESULTS_DIR, exist_ok=True)
@ -380,8 +367,8 @@ def run_optimization_loop(callback=None, config_override=None):
history = load_iteration_history() history = load_iteration_history()
start_iter = len(history) + 1 start_iter = len(history) + 1
best_sharpe = max((h["sharpe"] for h in history), default=0) best_score = max((h.get("cost_improvement", 0) for h in history), default=0)
update_status(best_sharpe=best_sharpe) update_status(best_score=best_score)
setup_windows_remote() setup_windows_remote()
scp_to_windows(os.path.join(BASE_DIR, "ml_engine", "train_and_backtest.py"), "train_and_backtest.py") scp_to_windows(os.path.join(BASE_DIR, "ml_engine", "train_and_backtest.py"), "train_and_backtest.py")
@ -418,23 +405,26 @@ def run_optimization_loop(callback=None, config_override=None):
with open(results_local) as f: with open(results_local) as f:
results = json.load(f) results = json.load(f)
current_sharpe = results.get("sharpe_ratio", 0) current_score = results.get("cost_basis_improvement_pct", 0)
is_best = current_sharpe > best_sharpe signal_count = results.get("strong_buy_signal_count", 0)
is_best = current_score > best_score and signal_count >= MIN_SIGNAL_COUNT
if is_best: if is_best:
best_sharpe = current_sharpe best_score = current_score
with open(best_config_path, "w") as f: with open(best_config_path, "w") as f:
json.dump(config, f, indent=2) json.dump(config, f, indent=2)
update_status(best_sharpe=best_sharpe) update_status(best_score=best_score)
iter_data = { iter_data = {
"iteration": iteration, "iteration": iteration,
"timestamp": datetime.now(timezone.utc).isoformat(), "timestamp": datetime.now(timezone.utc).isoformat(),
"sharpe": current_sharpe, "cost_improvement": current_score,
"return": results.get("total_return_pct", 0), "signal_count": signal_count,
"max_drawdown": results.get("max_drawdown_pct", 0), "signal_frequency": results.get("signal_frequency_pct", 0),
"win_rate": results.get("win_rate", 0), "r2_score": results.get("model_r2_score", 0),
"trades": results.get("trade_count", 0), "score_at_bottoms": results.get("avg_score_at_actual_bottoms", 0),
"profit_factor": results.get("profit_factor", 0), "score_at_tops": results.get("avg_score_at_actual_tops", 0),
"quality": results.get("pct_quality_strong_buy", 0),
"model_type": config.get("model_type", "unknown"), "model_type": config.get("model_type", "unknown"),
"is_best": is_best, "is_best": is_best,
"config": config, "config": config,
@ -459,10 +449,10 @@ def run_optimization_loop(callback=None, config_override=None):
update_status(state="completed") update_status(state="completed")
return return
# LLM suggestion
try: try:
summary_history = [ summary_history = [
{k: h[k] for k in ("iteration", "sharpe", "return", "win_rate", "trades", "model_type")} {k: h[k] for k in ("iteration", "cost_improvement", "signal_count", "r2_score", "model_type")
if k in h}
for h in history for h in history
] ]
new_config, reasoning = analyze_and_suggest(config, results, summary_history) new_config, reasoning = analyze_and_suggest(config, results, summary_history)
@ -475,8 +465,8 @@ def run_optimization_loop(callback=None, config_override=None):
except Exception: except Exception:
import random import random
hp = config.get("hyperparameters", {}) hp = config.get("hyperparameters", {})
hp["learning_rate"] = hp.get("learning_rate", 0.05) * random.uniform(0.8, 1.2) hp["learning_rate"] = hp.get("learning_rate", 0.01) * random.uniform(0.8, 1.2)
hp["max_depth"] = max(3, min(10, hp.get("max_depth", 6) + random.choice([-1, 0, 1]))) hp["max_depth"] = max(3, min(10, hp.get("max_depth", 5) + random.choice([-1, 0, 1])))
config["hyperparameters"] = hp config["hyperparameters"] = hp
update_status(state="completed") update_status(state="completed")