bizzle/btc-accumulation-monitor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pivot: rewrite as BTC accumulation signal optimizer

Browse Source 
Replace day-trading bot with long-term accumulation signal model.
Predicts optimal BUY times using forward return analysis at 7d/30d/90d
horizons, scoring each candle 0-100. Primary metric is now
cost_basis_improvement_pct (model buy price vs DCA).

- train_and_backtest.py: regression models (XGBoost/LSTM hybrid),
  accumulation-focused features (price position, momentum, volatility,
  volume, cycle), forward return targets, signal quality backtesting
- orchestrator.py: cost improvement scoring, signal count validation
- analyzer.py: accumulation-focused LLM system prompt
- dashboard: cost improvement display, signal metrics table
- config: new accumulation-focused parameters

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
BizzleBot 2026-03-19 23:51:43 +00:00
parent a21e635d9f
commit 560863fa0d
5 changed files with 829 additions and 797 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
config/initial_config.json
View File

@ -1,62 +1,53 @@
{
"model_type": "hybrid",
"features": {
"technical_indicators": ["RSI_14", "RSI_7", "MACD_line", "MACD_signal", "MACD_hist", "BB_upper", "BB_lower", "BB_width", "ATR_14", "SMA_20", "SMA_50", "EMA_10", "EMA_20", "OBV", "stoch_k", "stoch_d", "williams_r", "CCI_20", "ROC_10"],
"lookback_periods": [3, 5, 10, 20],
"use_volume_features": true,
"use_volatility_features": true,
"use_candle_patterns": false,
"use_lag_features": true,
"lag_periods": [1, 2, 3, 5],
"use_price_position": true,
"use_momentum": true,
"use_volatility": true,
"use_volume": true,
"use_cycle": true,
"use_pca": true,
"pca_variance": 0.95,
"use_scaler": true
},
"target": {
"type": "classification",
"direction": "both",
"horizon_candles": 8,
"threshold_pct": 1.5
"type": "regression",
"forward_periods_1h": [168, 720, 2160],
"forward_periods_4h": [42, 180, 540],
"weights": [0.2, 0.3, 0.5],
"score_range": [0, 100]
},
"hyperparameters": {
"learning_rate": 0.001,
"learning_rate": 0.01,
"max_depth": 5,
"n_estimators": 300,
"n_estimators": 500,
"subsample": 0.8,
"colsample_bytree": 0.8,
"min_child_weight": 5,
"min_child_weight": 10,
"gamma": 0.3,
"reg_alpha": 0.1,
"reg_lambda": 2.0,
"reg_alpha": 0.5,
"reg_lambda": 3.0,
"lstm_hidden_size": 128,
"lstm_num_layers": 2,
"lstm_dropout": 0.3,
"lstm_epochs": 100,
"lstm_batch_size": 64,
"lstm_sequence_length": 20,
"lstm_sequence_length": 30,
"lstm_patience": 10
},
"strategy": {
"entry_threshold": 0.60,
"exit_type": "trailing_stop",
"stop_loss_pct": 2.0,
"take_profit_pct": 4.0,
"trailing_stop_pct": 1.5,
"position_sizing": "confidence_scaled",
"max_position_pct": 100,
"min_confidence_to_trade": 0.55,
"dynamic_sl_tp": true,
"atr_sl_multiplier": 1.5,
"atr_tp_multiplier": 3.0
"strong_buy_threshold": 80,
"good_buy_threshold": 70,
"poor_threshold": 30
},
"training": {
"rolling_window": true,
"rolling_train_size": 2500,
"rolling_test_size": 300,
"walk_forward_windows": 5,
"train_pct": 0.7,
"validation_pct": 0.15,
"test_pct": 0.15,
"rolling_window": true,
"rolling_train_size": 2000,
"rolling_test_size": 200
"test_pct": 0.15
},
"timeframe": "4h"
}
}

113
dashboard/server.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3
"""
BTC ML Trading Strategy Optimizer — Web Dashboard
BTC Accumulation Signal Optimizer -- Web Dashboard
FastAPI server with inline HTML/CSS/JS dashboard.
"""
@ -13,42 +13,35 @@ from fastapi import FastAPI
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
from pydantic import BaseModel
# Add project root to path
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, BASE_DIR)
import orchestrator
app = FastAPI(title="BTC ML Optimizer Dashboard")
app = FastAPI(title="BTC Accumulation Signal Optimizer")
CONFIG_DIR = os.path.join(BASE_DIR, "config")
RESULTS_DIR = os.path.join(BASE_DIR, "results")
ITERATIONS_LOG = os.path.join(RESULTS_DIR, "iterations.jsonl")
# Background thread reference
_opt_thread: threading.Thread | None = None
_opt_thread = None
class ConfigUpdate(BaseModel):
config: dict
# ── API Endpoints ──────────────────────────────────────────────
@app.get("/api/status")
def api_status():
status = orchestrator.get_status()
return status
return orchestrator.get_status()
@app.get("/api/iterations")
def api_iterations():
iterations = orchestrator.load_iteration_history()
# Strip heavy config from list view
slim = []
for it in iterations:
entry = {k: v for k, v in it.items() if k != "config"}
entry = {k: v for k, v in it.items() if k not in ("config", "results")}
slim.append(entry)
return slim
@ -94,11 +87,11 @@ def api_stop():
def api_best():
best_path = os.path.join(CONFIG_DIR, "best_config.json")
if not os.path.exists(best_path):
return {"config": None, "sharpe": 0}
return {"config": None, "best_score": 0}
with open(best_path) as f:
config = json.load(f)
iterations = orchestrator.load_iteration_history()
best_iter = max(iterations, key=lambda x: x.get("sharpe", 0)) if iterations else {}
best_iter = max(iterations, key=lambda x: x.get("cost_improvement", 0)) if iterations else {}
return {"config": config, "best_iteration": best_iter}
@ -117,15 +110,12 @@ def api_download_best_config():
return JSONResponse({"error": "No best config yet"}, status_code=404)
# ── Dashboard HTML ─────────────────────────────────────────────
DASHBOARD_HTML = """<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>BTC ML Optimizer</title>
<title>BTC Accumulation Signal Optimizer</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet">
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.4/dist/chart.umd.min.js"></script>
@ -138,7 +128,6 @@ h1{font-size:1.5rem;font-weight:700;display:flex;align-items:center;gap:10px}
h1 .btc{color:var(--accent);font-size:1.8rem}
h2{font-size:1rem;font-weight:600;color:var(--text-dim);margin-bottom:12px;text-transform:uppercase;letter-spacing:.05em;font-size:.8rem}
/* Header */
.header{display:flex;justify-content:space-between;align-items:center;padding:16px 0;border-bottom:1px solid var(--border);margin-bottom:16px;flex-wrap:wrap;gap:12px}
.controls{display:flex;gap:8px;align-items:center}
.btn{padding:8px 18px;border:none;border-radius:6px;font-family:inherit;font-weight:600;font-size:.85rem;cursor:pointer;transition:all .15s}
@ -147,7 +136,6 @@ h2{font-size:1rem;font-weight:600;color:var(--text-dim);margin-bottom:12px;text-
.btn-secondary{background:var(--border);color:var(--text)}.btn-secondary:hover{background:var(--card-hover)}
.btn:disabled{opacity:.4;cursor:not-allowed}
/* Status badge */
.status-badge{display:inline-flex;align-items:center;gap:6px;padding:4px 12px;border-radius:20px;font-size:.8rem;font-weight:600}
.status-idle{background:#1e3a5f;color:#60a5fa}
.status-running{background:#1a3a2a;color:var(--green)}
@ -156,19 +144,16 @@ h2{font-size:1rem;font-weight:600;color:var(--text-dim);margin-bottom:12px;text-
.pulse{width:8px;height:8px;border-radius:50%;background:currentColor;animation:pulse 1.5s infinite}
@keyframes pulse{0%,100%{opacity:1}50%{opacity:.3}}
/* Best Sharpe display */
.best-sharpe{text-align:right}
.best-sharpe .label{font-size:.7rem;text-transform:uppercase;letter-spacing:.1em;color:var(--text-dim)}
.best-sharpe .value{font-size:2.2rem;font-weight:700;color:var(--accent);font-family:var(--mono)}
.best-score{text-align:right}
.best-score .label{font-size:.7rem;text-transform:uppercase;letter-spacing:.1em;color:var(--text-dim)}
.best-score .value{font-size:2.2rem;font-weight:700;color:var(--accent);font-family:var(--mono)}
.best-score .unit{font-size:1rem;color:var(--text-dim)}
/* Grid layout */
.grid{display:grid;grid-template-columns:1fr 360px;gap:16px}
@media(max-width:900px){.grid{grid-template-columns:1fr}}
/* Cards */
.card{background:var(--card);border-radius:10px;padding:16px;border:1px solid var(--border)}
/* Iteration table */
.table-wrap{overflow-x:auto;max-height:400px;overflow-y:auto}
table{width:100%;border-collapse:collapse;font-size:.82rem}
th{position:sticky;top:0;background:var(--card);text-align:left;padding:8px 10px;color:var(--text-dim);font-weight:600;border-bottom:2px solid var(--border);font-size:.75rem;text-transform:uppercase;letter-spacing:.04em}
@ -177,15 +162,12 @@ tr.best-row{background:rgba(34,197,94,.1)}
tr.best-row td:first-child{border-left:3px solid var(--green)}
tr:hover{background:var(--card-hover)}
/* Chart */
.chart-container{position:relative;height:260px}
/* LLM panel */
.llm-panel{max-height:500px;overflow-y:auto}
.llm-entry{padding:10px;border-bottom:1px solid var(--border);font-size:.82rem;line-height:1.5}
.llm-entry .iter-label{font-weight:600;color:var(--accent);font-size:.75rem;margin-bottom:4px}
/* Config editor */
.config-section{margin-top:16px}
.config-toggle{cursor:pointer;user-select:none;display:flex;align-items:center;gap:6px}
.config-toggle .arrow{transition:transform .2s;font-size:.7rem}
@ -195,22 +177,19 @@ tr:hover{background:var(--card-hover)}
textarea.config-editor{width:100%;height:300px;background:var(--bg);color:var(--text);border:1px solid var(--border);border-radius:6px;padding:12px;font-family:var(--mono);font-size:.8rem;resize:vertical}
.config-actions{display:flex;gap:8px;margin-top:8px}
/* Downloads */
.downloads{display:flex;gap:8px;margin-top:16px;flex-wrap:wrap}
.downloads a{color:var(--accent);text-decoration:none;font-size:.82rem;padding:6px 12px;border:1px solid var(--accent);border-radius:6px;transition:all .15s}
.downloads a:hover{background:var(--accent);color:#000}
/* Footer */
.footer{text-align:center;color:var(--text-dim);font-size:.75rem;padding:20px 0;margin-top:16px;border-top:1px solid var(--border)}
</style>
</head>
<body>
<div class="container">
<!-- Header -->
<div class="header">
<div>
<h1><span class="btc">&#x20BF;</span> ML Strategy Optimizer</h1>
<h1><span class="btc">&#x20BF;</span> Accumulation Signal Optimizer</h1>
<div style="margin-top:8px">
<span id="statusBadge" class="status-badge status-idle"><span class="pulse"></span> Idle</span>
</div>
@ -220,47 +199,41 @@ textarea.config-editor{width:100%;height:300px;background:var(--bg);color:var(--
<button id="btnStart" class="btn btn-start" onclick="startOpt()">Start Optimization</button>
<button id="btnStop" class="btn btn-stop" onclick="stopOpt()" disabled>Stop</button>
</div>
<div class="best-sharpe">
<div class="label">Best Sharpe Ratio</div>
<div class="value" id="bestSharpe">0.000</div>
<div class="best-score">
<div class="label">Best Cost Improvement</div>
<div class="value" id="bestScore">0.0<span class="unit">%</span></div>
</div>
</div>
</div>
<!-- Main grid -->
<div class="grid">
<div class="left">
<!-- Iteration Table -->
<div class="card" style="margin-bottom:16px">
<h2>Iterations</h2>
<div class="table-wrap">
<table>
<thead>
<tr><th>#</th><th>Sharpe</th><th>Return%</th><th>MaxDD%</th><th>WinRate</th><th>Trades</th><th>PF</th><th>Model</th></tr>
<tr><th>#</th><th>Cost Imp%</th><th>Signals</th><th>Frequency</th><th>R2</th><th>Bottoms</th><th>Tops</th><th>Model</th></tr>
</thead>
<tbody id="iterBody"></tbody>
</table>
</div>
</div>
<!-- Equity Curve Chart -->
<div class="card">
<h2>Performance Over Iterations</h2>
<h2>Cost Improvement Over Iterations</h2>
<div class="chart-container">
<canvas id="sharpeChart"></canvas>
<canvas id="mainChart"></canvas>
</div>
</div>
</div>
<div class="right">
<!-- LLM Analysis -->
<div class="card" style="margin-bottom:16px">
<h2>LLM Analysis</h2>
<div class="llm-panel" id="llmPanel">
<div style="color:var(--text-dim);font-size:.82rem;padding:10px">No suggestions yet.</div>
</div>
</div>
<!-- Downloads -->
<div class="card">
<h2>Downloads</h2>
<div class="downloads">
@ -271,7 +244,6 @@ textarea.config-editor{width:100%;height:300px;background:var(--bg);color:var(--
</div>
</div>
<!-- Config Editor (collapsible) -->
<div class="card config-section">
<div class="config-toggle" id="configToggle" onclick="toggleConfig()">
<span class="arrow">&#9654;</span>
@ -286,22 +258,21 @@ textarea.config-editor{width:100%;height:300px;background:var(--bg);color:var(--
</div>
</div>
<div class="footer">BTC ML Trading Strategy Optimizer &mdash; VPS &rarr; Windows GPU &rarr; Mac Mini LLM</div>
<div class="footer">BTC Accumulation Signal Optimizer &mdash; VPS &rarr; Windows GPU &rarr; Mac Mini LLM</div>
</div>
<script>
let chart = null;
let pollInterval = null;
// Init chart
function initChart() {
const ctx = document.getElementById('sharpeChart').getContext('2d');
const ctx = document.getElementById('mainChart').getContext('2d');
chart = new Chart(ctx, {
type: 'line',
data: {
labels: [],
datasets: [{
label: 'Sharpe Ratio',
label: 'Cost Improvement %',
data: [],
borderColor: '#f7931a',
backgroundColor: 'rgba(247,147,26,0.1)',
@ -311,7 +282,7 @@ function initChart() {
pointRadius: 4,
pointBackgroundColor: '#f7931a'
}, {
label: 'Return %',
label: 'Signal Count',
data: [],
borderColor: '#22c55e',
borderWidth: 1.5,
@ -331,8 +302,8 @@ function initChart() {
},
scales: {
x: { ticks: { color: '#94a3b8' }, grid: { color: '#1e293b' } },
y: { position: 'left', ticks: { color: '#f7931a' }, grid: { color: '#1e293b' }, title: { display: true, text: 'Sharpe', color: '#f7931a' } },
y1: { position: 'right', ticks: { color: '#22c55e' }, grid: { drawOnChartArea: false }, title: { display: true, text: 'Return %', color: '#22c55e' } }
y: { position: 'left', ticks: { color: '#f7931a' }, grid: { color: '#1e293b' }, title: { display: true, text: 'Cost Improvement %', color: '#f7931a' } },
y1: { position: 'right', ticks: { color: '#22c55e' }, grid: { drawOnChartArea: false }, title: { display: true, text: 'Signal Count', color: '#22c55e' } }
}
}
});
@ -353,7 +324,7 @@ function updateStatusBadge(status) {
document.getElementById('btnStart').disabled = (state === 'running');
document.getElementById('btnStop').disabled = (state !== 'running');
document.getElementById('bestSharpe').textContent = (status.best_sharpe || 0).toFixed(3);
document.getElementById('bestScore').innerHTML = (status.best_score || 0).toFixed(1) + '<span class="unit">%</span>';
}
function updateIterations(iterations) {
@ -362,24 +333,23 @@ function updateIterations(iterations) {
tbody.innerHTML = '<tr><td colspan="8" style="color:var(--text-dim);text-align:center">No iterations yet</td></tr>';
return;
}
const bestSharpe = Math.max(...iterations.map(i => i.sharpe || 0));
const bestCI = Math.max(...iterations.map(i => i.cost_improvement || 0));
let html = '';
for (const it of iterations) {
const isBest = it.sharpe === bestSharpe && bestSharpe > 0;
const sc = it.sharpe > 1.5 ? 'var(--green)' : it.sharpe > 1.0 ? 'var(--yellow)' : 'var(--red)';
const isBest = it.cost_improvement === bestCI && bestCI > 0;
const sc = it.cost_improvement > 15 ? 'var(--green)' : it.cost_improvement > 10 ? 'var(--yellow)' : 'var(--red)';
html += '<tr class="' + (isBest ? 'best-row' : '') + '">';
html += '<td>' + it.iteration + '</td>';
html += '<td style="color:' + sc + ';font-weight:600">' + (it.sharpe||0).toFixed(3) + '</td>';
html += '<td>' + (it["return"]||0).toFixed(1) + '</td>';
html += '<td>' + (it.max_drawdown||0).toFixed(1) + '</td>';
html += '<td>' + ((it.win_rate||0)*100).toFixed(1) + '%</td>';
html += '<td>' + (it.trades||0) + '</td>';
html += '<td>' + (it.profit_factor||0).toFixed(2) + '</td>';
html += '<td>' + (it.model_type||'—') + '</td>';
html += '<td style="color:' + sc + ';font-weight:600">' + (it.cost_improvement||0).toFixed(1) + '</td>';
html += '<td>' + (it.signal_count||0) + '</td>';
html += '<td>' + (it.signal_frequency||0).toFixed(1) + '%</td>';
html += '<td>' + (it.r2_score||0).toFixed(4) + '</td>';
html += '<td>' + (it.score_at_bottoms||0).toFixed(1) + '</td>';
html += '<td>' + (it.score_at_tops||0).toFixed(1) + '</td>';
html += '<td>' + (it.model_type||'-') + '</td>';
html += '</tr>';
}
tbody.innerHTML = html;
// auto-scroll to bottom
const wrap = tbody.closest('.table-wrap');
wrap.scrollTop = wrap.scrollHeight;
}
@ -387,8 +357,8 @@ function updateIterations(iterations) {
function updateChart(iterations) {
if (!chart || !iterations.length) return;
chart.data.labels = iterations.map(i => '#' + i.iteration);
chart.data.datasets[0].data = iterations.map(i => i.sharpe || 0);
chart.data.datasets[1].data = iterations.map(i => i["return"] || 0);
chart.data.datasets[0].data = iterations.map(i => i.cost_improvement || 0);
chart.data.datasets[1].data = iterations.map(i => i.signal_count || 0);
chart.update('none');
}
@ -468,14 +438,9 @@ async function updateConfig() {
async function resetConfig() {
if (!confirm('Reset to initial config?')) return;
try {
const r = await fetch('/api/config');
// Fetch the initial config by reading it — for now just reload
location.reload();
} catch(e) { alert(e); }
try { location.reload(); } catch(e) { alert(e); }
}
// Init
initChart();
poll();
pollInterval = setInterval(poll, 10000);

213
llm_client/analyzer.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3
"""
LLM Strategy Analyzer -- Calls Ollama on Mac Mini to analyze results
LLM Accumulation Signal Analyzer -- Calls Ollama on Mac Mini to analyze results
and suggest config modifications for the next iteration.
"""
@ -11,121 +11,130 @@ import requests
OLLAMA_URL = "http://100.100.242.21:11434"
MODEL = "qwen3.5:27b"
SYSTEM_PROMPT = """You are a quantitative trading strategy optimizer. You analyze ML model backtesting results for a BTC/USDT trading strategy and suggest precise modifications to improve performance.
SYSTEM_PROMPT = """You are a quantitative analyst optimizing a BTC ACCUMULATION SIGNAL model. The goal is NOT day-trading -- it is finding statistically optimal times to BUY BTC for long-term holding.
## Your Task
Given the current configuration and results, suggest 1-3 specific, justified changes to the configuration for the next iteration. Be methodical and scientific -- change one thing at a time when possible.
## Core Question
"Given current market conditions, is NOW a good time to BUY BTC for long-term holding?"
## What the Model Does
For each candle, the model predicts an Accumulation Score (0-100):
- 90-100: STRONG BUY -- historically rare, excellent entry point
- 70-89: GOOD BUY -- better than average entry
- 50-69: NEUTRAL -- average time to buy
- 30-49: WAIT -- price likely to come down
- 0-29: POOR -- historically bad time to buy (near local tops)
The model is trained on ACTUAL forward returns at 7d, 30d, and 90d horizons, weighted 20/30/50. Times when buying led to the best long-term returns get the highest scores.
## Primary Metric: cost_basis_improvement_pct
This measures how much better the model's average buy price is vs uniform DCA.
- 10%+ = good
- 15%+ = excellent
- 20%+ = exceptional
Also require strong_buy_signal_count >= 30 for statistical validity.
## Config Parameters You Can Modify
**model_type**: "xgboost", "lightgbm", "catboost", "ensemble", "lstm", or "hybrid"
- xgboost: Generally best for structured data, fast GPU training
- lightgbm: Faster training, good with large feature sets
- catboost: Handles feature interactions well, less tuning needed
- ensemble: Combines xgboost+lightgbm+catboost, reduces variance but slower
- lstm: PyTorch LSTM neural network, captures temporal/sequential patterns in price data
- hybrid: Combines LSTM (60% weight) + XGBoost (40% weight). Only enters trades when BOTH models agree on direction. The hybrid model typically outperforms single models -- LSTM captures temporal patterns while XGBoost handles feature interactions. Recommended as default.
**model_type**: "xgboost", "lightgbm", "catboost", "lstm", or "hybrid"
- hybrid: Average of LSTM + XGBoost regression predictions. Recommended default.
- xgboost: Fast GPU training, good for structured features.
- lstm: Captures temporal patterns in price sequences.
**hyperparameters** (gradient boosting):
- learning_rate (0.001-0.3): Lower = more robust but slower. If overfitting, decrease.
- max_depth (3-10): Controls model complexity. Deeper = more overfitting risk.
- n_estimators (100-2000): More trees = better fit but diminishing returns.
- subsample (0.5-1.0): Row sampling. Lower = more regularization.
- colsample_bytree (0.5-1.0): Feature sampling per tree. Lower = more diversity.
- min_child_weight (1-20): Higher = more conservative splits.
- gamma (0-5): Minimum loss reduction for split. Higher = more pruning.
- reg_alpha (0-10): L1 regularization. Encourages sparsity.
- reg_lambda (0-10): L2 regularization. Prevents large weights.
- learning_rate (0.001-0.1): Lower = more robust. Start conservative.
- max_depth (3-8): Controls complexity. Deeper risks overfitting.
- n_estimators (200-1500): More trees = better fit but diminishing returns.
- subsample (0.5-1.0): Row sampling for regularization.
- colsample_bytree (0.5-1.0): Feature sampling per tree.
- min_child_weight (5-30): Higher = more conservative (important for noisy targets).
- gamma (0-5): Minimum loss reduction for split.
- reg_alpha (0-10): L1 regularization.
- reg_lambda (1-10): L2 regularization. Higher values prevent overfitting.
**hyperparameters** (LSTM-specific, used by lstm and hybrid model_types):
- lstm_hidden_size (32-256): LSTM hidden units. Larger = more capacity but overfitting risk. Default 128.
- lstm_num_layers (1-4): Stacked LSTM layers. 2 is usually optimal. More layers need more data.
- lstm_dropout (0.1-0.5): Dropout between LSTM layers and before output. Higher = more regularization.
- lstm_epochs (50-200): Max training epochs. Early stopping usually triggers before this.
- lstm_batch_size (32-128): Training batch size. Smaller = noisier gradients but better generalization.
- lstm_sequence_length (10-50): How many past candles the LSTM sees per prediction. Longer = more context but more memory. Default 20.
- lstm_patience (5-20): Early stopping patience on validation loss. Lower = stop sooner.
**hyperparameters** (LSTM):
- lstm_hidden_size (32-256): Hidden units.
- lstm_num_layers (1-4): Stacked layers. 2 is usually optimal.
- lstm_dropout (0.1-0.5): Regularization.
- lstm_epochs (50-200): Max training epochs (early stopping usually triggers).
- lstm_batch_size (32-128): Smaller = noisier but better generalization.
- lstm_sequence_length (15-60): Past candles the LSTM sees. Longer = more context.
- lstm_patience (5-20): Early stopping patience.
**target**:
- direction: "long", "short", or "both"
- horizon_candles (1-20): How far ahead to predict. Longer = smoother but lagging.
- threshold_pct (0.3-3.0): Minimum move % to label as positive. Higher = fewer but clearer signals.
- forward_periods_4h: List of 3 forward periods in 4h candles [short, medium, long].
Defaults: [42, 180, 540] = roughly [7d, 30d, 90d]
- weights: Weights for each period. Default [0.2, 0.3, 0.5] (emphasize long-term).
- score_range: [0, 100] -- do not change.
**strategy**:
- entry_threshold (0.5-0.8): Min prediction probability to enter trade. Higher = fewer trades, higher quality.
- stop_loss_pct (0.5-5.0): Max loss before exit (used when dynamic_sl_tp is false).
- take_profit_pct (1.0-10.0): Target profit (used when dynamic_sl_tp is false). Should be > stop_loss for positive expectancy.
- trailing_stop_pct (0.5-3.0): Trailing stop distance. Tighter = locks profit faster but exits early.
- min_confidence_to_trade (0.5-0.9): Absolute minimum confidence to consider.
- exit_type: "trailing_stop" or "fixed" (just SL/TP)
- dynamic_sl_tp (true/false): Use ATR-based dynamic stop-loss and take-profit instead of fixed percentages. Adapts to current volatility. Recommended: true.
- atr_sl_multiplier (1.0-3.0): ATR multiplier for stop-loss. E.g., 1.5 means SL = 1.5 * ATR(14). Lower = tighter stops.
- atr_tp_multiplier (2.0-5.0): ATR multiplier for take-profit. E.g., 3.0 means TP = 3.0 * ATR(14). Should be > atr_sl_multiplier.
- strong_buy_threshold (70-95): Score above which = STRONG BUY signal. Higher = fewer but better signals.
- good_buy_threshold (50-80): Score above which = GOOD BUY. Used for cost basis comparison.
- poor_threshold (10-40): Score below which = POOR time to buy.
**features**:
- use_volume_features (true/false): Volume features can be noisy in crypto.
- use_candle_patterns (true/false): Candle patterns may or may not help.
- use_lag_features (true/false): Lagged features capture momentum.
- lag_periods: List of lag periods [1,2,3,5,10]
- lookback_periods: List of lookback windows [3,5,10,20]
- use_scaler (true/false): Apply StandardScaler normalization to all features. Critical for LSTM, also helps gradient boosting. Recommended: true.
- use_pca (true/false): Apply PCA dimensionality reduction after scaling. Reduces noise and multicollinearity. Recommended with many features.
- pca_variance (0.80-0.99): Fraction of variance to retain with PCA. 0.95 keeps 95% of information. Lower = fewer dimensions, more noise removed.
- use_price_position (true/false): Distance from ATH, 52w high/low, percentile.
- use_momentum (true/false): RSI, MACD, Stochastic, Williams %R, ROC.
- use_volatility (true/false): Bollinger Bands, ATR, consecutive red candles, drawdown.
- use_volume (true/false): Volume ratio, OBV, red/green volume ratio.
- use_cycle (true/false): MA cross regime, candles since major drawdown.
- use_pca (true/false): PCA dimensionality reduction.
- pca_variance (0.80-0.99): Variance to retain.
- use_scaler (true/false): StandardScaler. Critical for LSTM.
**training**:
- walk_forward_windows (3-10): More windows = more robust but less data per window. Used when rolling_window is false.
- rolling_window (true/false): Use rolling window instead of static walk-forward splits. Trains on last N candles, tests on next M, slides forward. More realistic for time series. Recommended: true.
- rolling_train_size (1000-5000): Number of candles in the rolling training window. Larger = more data but older patterns.
- rolling_test_size (100-500): Number of candles in the rolling test window. Smaller = more retraining, better adaptation.
- rolling_window (true/false): Rolling vs static walk-forward.
- rolling_train_size (1500-5000): Training window candles.
- rolling_test_size (100-500): Test window candles.
## Key Metrics to Optimize (in priority order)
1. **Sharpe Ratio** (target: > 2.0): Risk-adjusted return. Most important metric.
2. **Profit Factor** (target: > 1.5): Gross profit / gross loss.
3. **Max Drawdown** (target: > -15%): Worst peak-to-trough decline.
4. **Win Rate** (target: > 55%): Percentage of winning trades.
5. **Trade Count**: Need enough trades for statistical significance (>50).
## Key Metrics to Analyze
1. **cost_basis_improvement_pct**: PRIMARY metric. How much better is model buy price vs DCA.
2. **strong_buy_signal_count**: Must be >= 30 for validity. Too few = raise threshold. Too many = lower it.
3. **signal_frequency_pct**: Should be 5-15%. If outside, adjust thresholds.
4. **avg_score_at_actual_bottoms**: Should be high (>70). Model should recognize bottoms.
5. **avg_score_at_actual_tops**: Should be low (<30). Model should avoid tops.
6. **model_r2_score**: Regression fit quality. > 0.2 is decent for financial data.
7. **per_window_cost_improvement**: Consistency across windows. Low variance = robust.
## Decision Guidelines
- If Sharpe < 1.0: The strategy is not working well. Consider larger changes (switch to hybrid, enable PCA/scaler, adjust target).
- If Sharpe 1.0-1.5: Decent. Fine-tune hyperparameters and thresholds.
- If Sharpe 1.5-2.0: Good. Make small, targeted improvements.
- If Sharpe > 2.0: Very good. Be careful not to overfit.
- If win_rate < 0.50 but profit_factor > 1.5: Strategy relies on big wins -- ok, tighten SL.
- If win_rate > 0.60 but profit_factor < 1.2: Many small wins but losses are too big -- widen TP or tighten SL.
- If trade_count < 30: Not enough trades. Lower entry_threshold or min_confidence.
- If max_drawdown < -20%: Too risky. Increase regularization, tighten stop loss, enable dynamic_sl_tp.
- If per_window_sharpe has high variance: Model is not stable. More regularization, enable PCA, or try hybrid.
- Check feature_importances: If top features make financial sense, good. If random features dominate, possible overfitting -- enable PCA or reduce features.
- For LSTM/hybrid: if underfitting, increase lstm_hidden_size or lstm_num_layers. If overfitting, increase lstm_dropout or decrease lstm_sequence_length.
- The hybrid model combining LSTM + XGBoost typically outperforms single models. LSTM captures temporal patterns while XGBoost handles feature interactions. Use hybrid as the default unless you have a specific reason not to.
- If cost_improvement < 5%: Strategy is barely working. Try: switch model type, enable all features, increase training window, lower good_buy_threshold.
- If cost_improvement 5-10%: Decent. Fine-tune thresholds and hyperparameters.
- If cost_improvement 10-15%: Good. Make targeted improvements -- focus on signal consistency.
- If cost_improvement > 15%: Very good. Be careful not to overfit. Check per_window variance.
- If signal_count < 30: Not statistically valid. Lower strong_buy_threshold, increase training data.
- If signal_frequency > 20%: Too many signals = not selective enough. Raise threshold.
- If signal_frequency < 3%: Too few signals. Lower threshold.
- If score_at_bottoms < 60: Model is missing bottoms. More features, different model type.
- If score_at_tops > 40: Model is not avoiding tops. More regularization.
- If per_window has high variance: Model is unstable. Increase regularization, try hybrid.
- Check feature_importances: price position features should dominate (distance from ATH, percentile).
## Response Format
You MUST respond with ONLY a JSON object (no markdown, no explanation outside the JSON):
```
{
"reasoning": "Explanation of what you observed and why you're making these changes",
"reasoning": "Explanation of observations and why you are making these changes",
"changes": ["Change 1 description", "Change 2 description"],
"config": { <complete modified config JSON> }
}
```
The "config" field must contain the COMPLETE config (not just changes) so it can be used directly."""
The "config" field must contain the COMPLETE config so it can be used directly."""
def analyze_and_suggest(current_config: dict, results: dict,
iteration_history: list = None) -> tuple[dict, str]:
def analyze_and_suggest(current_config, results, iteration_history=None):
"""
Send current results to LLM and get suggested config modifications.
Returns (new_config, reasoning).
"""
# Build the user prompt with context
history_text = ""
if iteration_history:
history_text = "\n## Previous Iterations (most recent last)\n"
for h in iteration_history[-5:]:
history_text += (
f"- Iteration {h['iteration']}: Sharpe={h['sharpe']}, "
f"Return={h['return']}%, WinRate={h['win_rate']}, "
f"Trades={h['trades']}, Model={h['model_type']}\n"
f"- Iteration {h.get('iteration', '?')}: "
f"CostImprovement={h.get('cost_improvement', 0):.1f}%, "
f"Signals={h.get('signal_count', 0)}, "
f"R2={h.get('r2_score', 0):.4f}, "
f"Model={h.get('model_type', '?')}\n"
)
user_prompt = f"""## Current Configuration
@ -134,21 +143,24 @@ def analyze_and_suggest(current_config: dict, results: dict,
```
## Current Results
- Sharpe Ratio: {results.get('sharpe_ratio', 0)}
- Total Return: {results.get('total_return_pct', 0)}%
- Max Drawdown: {results.get('max_drawdown_pct', 0)}%
- Win Rate: {results.get('win_rate', 0)}
- Trade Count: {results.get('trade_count', 0)}
- Profit Factor: {results.get('profit_factor', 0)}
- Avg Trade Duration: {results.get('avg_trade_duration_candles', 0)} candles
- Per-Window Sharpe: {results.get('per_window_sharpe', [])}
- Cost Basis Improvement: {results.get('cost_basis_improvement_pct', 0):.1f}%
- Avg Cost (Model): ${results.get('avg_cost_basis_model', 0):,.2f}
- Avg Cost (DCA): ${results.get('avg_cost_basis_dca', 0):,.2f}
- Strong Buy Signals: {results.get('strong_buy_signal_count', 0)}
- Good Buy Signals: {results.get('good_buy_signal_count', 0)}
- Signal Frequency: {results.get('signal_frequency_pct', 0):.1f}%
- Quality of Strong Buys: {results.get('pct_quality_strong_buy', 0):.1%}
- Model R2: {results.get('model_r2_score', 0):.4f}
- Score at Actual Bottoms: {results.get('avg_score_at_actual_bottoms', 0):.1f}
- Score at Actual Tops: {results.get('avg_score_at_actual_tops', 0):.1f}
- Per-Window Improvement: {results.get('per_window_cost_improvement', [])}
- Score Distribution: {results.get('score_distribution', {})}
## Top Feature Importances
{json.dumps(dict(list(results.get('feature_importances', {}).items())[:15]), indent=2)}
{history_text}
Analyze these results and suggest 1-3 specific modifications to the config. Return ONLY valid JSON."""
# Call Ollama
payload = {
"model": MODEL,
"messages": [
@ -168,7 +180,6 @@ Analyze these results and suggest 1-3 specific modifications to the config. Retu
resp.raise_for_status()
content = resp.json()["message"]["content"]
# Parse JSON from response (handle markdown code blocks)
# Strip thinking tags if present
content = re.sub(r"<think>.*?</think>", "", content, flags=re.DOTALL).strip()
@ -176,8 +187,6 @@ Analyze these results and suggest 1-3 specific modifications to the config. Retu
if json_match:
parsed = json.loads(json_match.group(1))
else:
# Try parsing the whole response as JSON
# Find the outermost JSON object
brace_start = content.find("{")
if brace_start >= 0:
depth = 0
@ -198,7 +207,6 @@ Analyze these results and suggest 1-3 specific modifications to the config. Retu
changes = parsed.get("changes", [])
new_config = parsed.get("config", current_config)
# Validate that config has required fields
required_keys = ["model_type", "features", "target", "hyperparameters", "strategy", "training"]
for key in required_keys:
if key not in new_config:
@ -209,22 +217,25 @@ Analyze these results and suggest 1-3 specific modifications to the config. Retu
if __name__ == "__main__":
# Test with dummy data
import sys
config_path = sys.argv[1] if len(sys.argv) > 1 else "config/initial_config.json"
with open(config_path) as f:
config = json.load(f)
dummy_results = {
"sharpe_ratio": 1.2,
"total_return_pct": 15.3,
"max_drawdown_pct": -12.5,
"win_rate": 0.55,
"trade_count": 120,
"profit_factor": 1.4,
"avg_trade_duration_candles": 7.2,
"feature_importances": {"RSI_14": 0.15, "MACD_hist": 0.12, "BB_width": 0.10},
"per_window_sharpe": [1.0, 1.3, 1.5, 0.9, 1.1],
"cost_basis_improvement_pct": 8.5,
"avg_cost_basis_model": 65000,
"avg_cost_basis_dca": 71000,
"strong_buy_signal_count": 45,
"good_buy_signal_count": 120,
"signal_frequency_pct": 7.2,
"pct_quality_strong_buy": 0.72,
"model_r2_score": 0.22,
"avg_score_at_actual_bottoms": 68.5,
"avg_score_at_actual_tops": 35.2,
"per_window_cost_improvement": [7.1, 9.3, 8.8, 10.2, 7.0],
"score_distribution": {"0-20": 80, "20-40": 150, "40-60": 200, "60-80": 130, "80-100": 40},
"feature_importances": {"dist_from_ath_pct": 0.18, "RSI_14": 0.12, "price_percentile_365": 0.10},
}
new_config, reasoning = analyze_and_suggest(config, dummy_results)

1057
ml_engine/train_and_backtest.py
View File

File diff suppressed because it is too large Load Diff

186
orchestrator.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3
"""
BTC ML Trading Strategy Optimizer — Orchestrator
BTC Accumulation Signal Optimizer -- Orchestrator
Coordinates the optimization loop across VPS, Windows PC (GPU), and Mac Mini (LLM).
"""
@ -28,7 +28,8 @@ MAC_MINI_HOST = "bizzle@bizzles-mac-mini-1"
MAX_ITERATIONS = 50
CONVERGENCE_WINDOW = 5
CONVERGENCE_THRESHOLD = 0.01 # 1% improvement
TARGET_SHARPE = 3.0
TARGET_COST_IMPROVEMENT = 20.0 # 20% cost basis improvement = exceptional
MIN_SIGNAL_COUNT = 30 # Minimum strong buy signals for valid results
ML_TIMEOUT = 600 # 10 minutes
# Colors
@ -98,7 +99,6 @@ def run_ml_training():
)
if result.returncode != 0:
raise RuntimeError(f"ML training failed:\n{result.stderr}\n{result.stdout}")
# Print training output
for line in result.stdout.strip().split("\n"):
log(f" {C.DIM}{line}", C.DIM)
return True
@ -127,45 +127,53 @@ def check_convergence(history):
if len(history) < CONVERGENCE_WINDOW + 1:
return False, "Not enough iterations"
recent = history[-CONVERGENCE_WINDOW:]
sharpes = [h["sharpe"] for h in recent]
# Only consider valid results (enough signals)
valid = [h for h in history if h.get("signal_count", 0) >= MIN_SIGNAL_COUNT]
# Check if best sharpe exceeds target
best_sharpe = max(h["sharpe"] for h in history)
if best_sharpe >= TARGET_SHARPE:
return True, f"Target Sharpe reached: {best_sharpe:.3f}"
if not valid:
return False, "No valid results yet"
recent = history[-CONVERGENCE_WINDOW:]
scores = [h.get("cost_improvement", 0) for h in recent]
# Check if best score exceeds target
best_score = max(h.get("cost_improvement", 0) for h in valid)
if best_score >= TARGET_COST_IMPROVEMENT:
return True, f"Target cost improvement reached: {best_score:.1f}%"
# Check if improvement has stalled
best_recent = max(sharpes)
worst_recent = min(sharpes)
best_recent = max(scores)
worst_recent = min(scores)
if best_recent > 0 and (best_recent - worst_recent) / best_recent < CONVERGENCE_THRESHOLD:
return True, f"Converged: Sharpe variance < {CONVERGENCE_THRESHOLD*100}% over {CONVERGENCE_WINDOW} iterations"
return True, f"Converged: variance < {CONVERGENCE_THRESHOLD*100}% over {CONVERGENCE_WINDOW} iterations"
return False, ""
def print_header():
print(f"""
{C.BOLD}{C.CYAN}╔══════════════════════════════════════════════════╗
║ BTC ML Trading Strategy Optimizer ║
║ VPS → Windows GPU → Mac Mini LLM → Loop ║
╚══════════════════════════════════════════════════╝{C.RESET}
{C.BOLD}{C.CYAN}========================================================
BTC Accumulation Signal Optimizer
VPS -> Windows GPU -> Mac Mini LLM -> Loop
========================================================{C.RESET}
""")
def print_results(results, iteration):
sharpe = results.get("sharpe_ratio", 0)
sharpe_color = C.GREEN if sharpe > 1.5 else C.YELLOW if sharpe > 1.0 else C.RED
cost_imp = results.get("cost_basis_improvement_pct", 0)
color = C.GREEN if cost_imp > 15 else C.YELLOW if cost_imp > 10 else C.RED
print(f"""
{C.BOLD}━━━ Iteration {iteration} Results ━━━{C.RESET}
Sharpe Ratio: {sharpe_color}{C.BOLD}{sharpe:.3f}{C.RESET}
Total Return: {results.get('total_return_pct', 0):.1f}%
Max Drawdown: {results.get('max_drawdown_pct', 0):.1f}%
Win Rate: {results.get('win_rate', 0):.1%}
Trade Count: {results.get('trade_count', 0)}
Profit Factor: {results.get('profit_factor', 0):.3f}
Avg Duration: {results.get('avg_trade_duration_candles', 0):.1f} candles
Window Sharpes: {results.get('per_window_sharpe', [])}
{C.BOLD}--- Iteration {iteration} Results ---{C.RESET}
Cost Improvement: {color}{C.BOLD}{cost_imp:.1f}%{C.RESET}
Avg Cost (Model): ${results.get('avg_cost_basis_model', 0):,.2f}
Avg Cost (DCA): ${results.get('avg_cost_basis_dca', 0):,.2f}
Strong Signals: {results.get('strong_buy_signal_count', 0)}
Signal Frequency: {results.get('signal_frequency_pct', 0):.1f}%
Quality Score: {results.get('pct_quality_strong_buy', 0):.1%}
Model R2: {results.get('model_r2_score', 0):.4f}
Score@Bottoms: {results.get('avg_score_at_actual_bottoms', 0):.1f}
Score@Tops: {results.get('avg_score_at_actual_tops', 0):.1f}
Window Improvements: {results.get('per_window_cost_improvement', [])}
""")
@ -173,14 +181,11 @@ def main():
print_header()
os.makedirs(RESULTS_DIR, exist_ok=True)
# Step 1: Ensure data
ensure_data()
# Step 2: Load or create initial config
config_path = os.path.join(CONFIG_DIR, "initial_config.json")
best_config_path = os.path.join(CONFIG_DIR, "best_config.json")
# Resume from best config if it exists
if os.path.exists(best_config_path):
log("Resuming from best_config.json", C.GREEN)
with open(best_config_path) as f:
@ -191,29 +196,24 @@ def main():
history = load_iteration_history()
start_iter = len(history) + 1
best_sharpe = max((h["sharpe"] for h in history), default=0)
best_score = max((h.get("cost_improvement", 0) for h in history), default=0)
log(f"Starting at iteration {start_iter}, best Sharpe so far: {best_sharpe:.3f}", C.BOLD)
log(f"Starting at iteration {start_iter}, best cost improvement so far: {best_score:.1f}%", C.BOLD)
# Step 3: Setup Windows remote
setup_windows_remote()
# SCP the ML engine script (once)
log("Uploading ML engine to Windows...", C.CYAN)
scp_to_windows(os.path.join(BASE_DIR, "ml_engine", "train_and_backtest.py"), "train_and_backtest.py")
# SCP data files (once)
for tf in ["1h", "4h"]:
data_file = os.path.join(DATA_DIR, f"btc_{tf}.csv")
if os.path.exists(data_file):
log(f"Uploading btc_{tf}.csv to Windows...", C.CYAN)
scp_to_windows(data_file, f"btc_{tf}.csv")
# Import LLM analyzer
sys.path.insert(0, os.path.join(BASE_DIR, "llm_client"))
from analyzer import analyze_and_suggest
# Main optimization loop
for iteration in range(start_iter, MAX_ITERATIONS + 1):
log(f"\n{'='*50}", C.BOLD)
log(f"ITERATION {iteration}/{MAX_ITERATIONS}", f"{C.BOLD}{C.CYAN}")
@ -222,13 +222,11 @@ def main():
f"Depth: {config.get('hyperparameters', {}).get('max_depth', '?')}", C.DIM)
log(f"{'='*50}", C.BOLD)
# Write current config to temp file and SCP
tmp_config = os.path.join(BASE_DIR, "config", "current_config.json")
with open(tmp_config, "w") as f:
json.dump(config, f, indent=2)
scp_to_windows(tmp_config, "config.json")
# Run ML training on Windows
try:
run_ml_training()
except (RuntimeError, subprocess.TimeoutExpired) as e:
@ -238,7 +236,6 @@ def main():
config = history[-1].get("config", config)
continue
# Fetch results from Windows
results_local = os.path.join(RESULTS_DIR, f"results_iter_{iteration}.json")
scp_from_windows("results.json", results_local)
@ -247,34 +244,35 @@ def main():
print_results(results, iteration)
# Track best
current_sharpe = results.get("sharpe_ratio", 0)
is_best = current_sharpe > best_sharpe
current_score = results.get("cost_basis_improvement_pct", 0)
signal_count = results.get("strong_buy_signal_count", 0)
is_best = current_score > best_score and signal_count >= MIN_SIGNAL_COUNT
if is_best:
best_sharpe = current_sharpe
best_score = current_score
with open(best_config_path, "w") as f:
json.dump(config, f, indent=2)
log(f"NEW BEST! Sharpe: {best_sharpe:.3f}", f"{C.BOLD}{C.GREEN}")
log(f"NEW BEST! Cost Improvement: {best_score:.1f}%", f"{C.BOLD}{C.GREEN}")
# Log iteration
iter_data = {
"iteration": iteration,
"timestamp": datetime.now(timezone.utc).isoformat(),
"sharpe": current_sharpe,
"return": results.get("total_return_pct", 0),
"max_drawdown": results.get("max_drawdown_pct", 0),
"win_rate": results.get("win_rate", 0),
"trades": results.get("trade_count", 0),
"profit_factor": results.get("profit_factor", 0),
"cost_improvement": current_score,
"avg_30d_return": results.get("avg_quality_score_strong_buy", 0),
"avg_90d_return": results.get("pct_quality_strong_buy", 0),
"signal_count": signal_count,
"signal_frequency": results.get("signal_frequency_pct", 0),
"r2_score": results.get("model_r2_score", 0),
"score_at_bottoms": results.get("avg_score_at_actual_bottoms", 0),
"score_at_tops": results.get("avg_score_at_actual_tops", 0),
"model_type": config.get("model_type", "unknown"),
"is_best": is_best,
"config": config,
"results": results,
}
save_iteration(iter_data)
history.append(iter_data)
# Check convergence
converged, reason = check_convergence(history)
if converged:
log(f"\nOptimization converged: {reason}", f"{C.BOLD}{C.GREEN}")
@ -284,17 +282,15 @@ def main():
log(f"\nMax iterations ({MAX_ITERATIONS}) reached.", C.YELLOW)
break
# Ask LLM for next config
log("\nConsulting LLM for strategy modifications...", C.MAGENTA)
try:
summary_history = [
{
"iteration": h["iteration"],
"sharpe": h["sharpe"],
"return": h["return"],
"win_rate": h["win_rate"],
"trades": h["trades"],
"model_type": h["model_type"],
"cost_improvement": h.get("cost_improvement", 0),
"signal_count": h.get("signal_count", 0),
"r2_score": h.get("r2_score", 0),
"model_type": h.get("model_type", "unknown"),
}
for h in history
]
@ -304,37 +300,34 @@ def main():
except Exception as e:
log(f"LLM call failed: {e}", C.RED)
log("Continuing with current config + random perturbation...", C.YELLOW)
# Small random perturbation as fallback
import random
hp = config.get("hyperparameters", {})
hp["learning_rate"] = hp.get("learning_rate", 0.05) * random.uniform(0.8, 1.2)
hp["max_depth"] = max(3, min(10, hp.get("max_depth", 6) + random.choice([-1, 0, 1])))
hp["learning_rate"] = hp.get("learning_rate", 0.01) * random.uniform(0.8, 1.2)
hp["max_depth"] = max(3, min(10, hp.get("max_depth", 5) + random.choice([-1, 0, 1])))
config["hyperparameters"] = hp
# Final summary
print(f"""
{C.BOLD}{C.GREEN}╔══════════════════════════════════════════════════╗
║ Optimization Complete! ║
╚══════════════════════════════════════════════════╝{C.RESET}
{C.BOLD}{C.GREEN}========================================================
Optimization Complete!
========================================================{C.RESET}
Total Iterations: {len(history)}
Best Sharpe: {C.BOLD}{best_sharpe:.3f}{C.RESET}
Best Config: {best_config_path}
Iteration Log: {ITERATIONS_LOG}
Total Iterations: {len(history)}
Best Cost Improvement: {C.BOLD}{best_score:.1f}%{C.RESET}
Best Config: {best_config_path}
Iteration Log: {ITERATIONS_LOG}
""")
# --- Library API for dashboard integration ---
# Shared state for dashboard
_stop_event = threading.Event()
_status = {
"state": "idle", # idle, running, completed, error
"state": "idle",
"iteration": 0,
"max_iterations": MAX_ITERATIONS,
"best_sharpe": 0.0,
"best_score": 0.0,
"error": None,
"llm_suggestions": [], # list of {iteration, reasoning, changes}
"llm_suggestions": [],
}
_status_lock = threading.Lock()
@ -352,15 +345,9 @@ def update_status(**kwargs):
def run_optimization_loop(callback=None, config_override=None):
"""
Run the optimization loop. Designed to be called from a background thread.
Args:
callback: Called after each iteration with (iteration_number, iter_data_dict).
config_override: Optional dict to use instead of loading from disk.
"""
"""Run the optimization loop from a background thread."""
_stop_event.clear()
update_status(state="running", iteration=0, error=None, best_sharpe=0.0)
update_status(state="running", iteration=0, error=None, best_score=0.0)
try:
os.makedirs(RESULTS_DIR, exist_ok=True)
@ -380,8 +367,8 @@ def run_optimization_loop(callback=None, config_override=None):
history = load_iteration_history()
start_iter = len(history) + 1
best_sharpe = max((h["sharpe"] for h in history), default=0)
update_status(best_sharpe=best_sharpe)
best_score = max((h.get("cost_improvement", 0) for h in history), default=0)
update_status(best_score=best_score)
setup_windows_remote()
scp_to_windows(os.path.join(BASE_DIR, "ml_engine", "train_and_backtest.py"), "train_and_backtest.py")
@ -418,23 +405,26 @@ def run_optimization_loop(callback=None, config_override=None):
with open(results_local) as f:
results = json.load(f)
current_sharpe = results.get("sharpe_ratio", 0)
is_best = current_sharpe > best_sharpe
current_score = results.get("cost_basis_improvement_pct", 0)
signal_count = results.get("strong_buy_signal_count", 0)
is_best = current_score > best_score and signal_count >= MIN_SIGNAL_COUNT
if is_best:
best_sharpe = current_sharpe
best_score = current_score
with open(best_config_path, "w") as f:
json.dump(config, f, indent=2)
update_status(best_sharpe=best_sharpe)
update_status(best_score=best_score)
iter_data = {
"iteration": iteration,
"timestamp": datetime.now(timezone.utc).isoformat(),
"sharpe": current_sharpe,
"return": results.get("total_return_pct", 0),
"max_drawdown": results.get("max_drawdown_pct", 0),
"win_rate": results.get("win_rate", 0),
"trades": results.get("trade_count", 0),
"profit_factor": results.get("profit_factor", 0),
"cost_improvement": current_score,
"signal_count": signal_count,
"signal_frequency": results.get("signal_frequency_pct", 0),
"r2_score": results.get("model_r2_score", 0),
"score_at_bottoms": results.get("avg_score_at_actual_bottoms", 0),
"score_at_tops": results.get("avg_score_at_actual_tops", 0),
"quality": results.get("pct_quality_strong_buy", 0),
"model_type": config.get("model_type", "unknown"),
"is_best": is_best,
"config": config,
@ -459,10 +449,10 @@ def run_optimization_loop(callback=None, config_override=None):
update_status(state="completed")
return
# LLM suggestion
try:
summary_history = [
{k: h[k] for k in ("iteration", "sharpe", "return", "win_rate", "trades", "model_type")}
{k: h[k] for k in ("iteration", "cost_improvement", "signal_count", "r2_score", "model_type")
if k in h}
for h in history
]
new_config, reasoning = analyze_and_suggest(config, results, summary_history)
@ -475,8 +465,8 @@ def run_optimization_loop(callback=None, config_override=None):
except Exception:
import random
hp = config.get("hyperparameters", {})
hp["learning_rate"] = hp.get("learning_rate", 0.05) * random.uniform(0.8, 1.2)
hp["max_depth"] = max(3, min(10, hp.get("max_depth", 6) + random.choice([-1, 0, 1])))
hp["learning_rate"] = hp.get("learning_rate", 0.01) * random.uniform(0.8, 1.2)
hp["max_depth"] = max(3, min(10, hp.get("max_depth", 5) + random.choice([-1, 0, 1])))
config["hyperparameters"] = hp
update_status(state="completed")