claude_dev
6ed71ed8cf
根因: vnpy run_optimization 孙进程 spawn re-import setting.py 重置 DB → load 0根 → 空 statistics;连带 task_id 不一致 + statistics 含 date/numpy 序列化失败 + 前端单页轮询 180s 超时。 修复(不动 vnpy 源码): - cta_optimizer: vt_setting.json 适配(vnpy 原生机制 setting.py:43 load_json,孙进程拿到正确 DB) + task_id 贯通 + 主聚合持久化(combos) + 过滤空 statistics 记录 - result_store: _json_default(date→isoformat + numpy→原生),save_result 两处 json.dumps 加 default - routes: optimization-results 改读 DB 主聚合, fallback 内存(重启不丢) - runner: _opt_worker 加 task_id 参数 + submit_optimize 传递(对齐 _cta_worker) - 前端: Optimize 提交后跳 progress;Progress 用 opt_ 前缀判断跳 optimize-result;新建 OptimizeResult 参数表格页;router 加路由 验证: 端到端 optimization-results http=200 + 4组合非空 statistics + 浏览器提交→跳 optimize-result 页表格渲染。
258 lines
10 KiB
Python
258 lines
10 KiB
Python
"""CTA strategy parameter optimization wrapper using vnpy_ctastrategy.backtesting."""
|
|
import sys
|
|
import os
|
|
import json
|
|
import logging
|
|
import traceback
|
|
import uuid
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import List, Any
|
|
|
|
# Add vnpy source to path for local development
|
|
_VNPY_SRC = os.path.join(os.path.dirname(__file__), "..", "vnpy_v4.4.0")
|
|
_VNPY_SRC = os.path.abspath(_VNPY_SRC)
|
|
if _VNPY_SRC not in sys.path:
|
|
sys.path.insert(0, _VNPY_SRC)
|
|
|
|
from sanguo_backtest.result_store import BacktestResult, save_result
|
|
from sanguo_backtest.cta_engine import guess_exchange, Exchange
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _ensure_vnpy_settings_for_spawn(cfg) -> None:
|
|
"""Write vt_setting.json so vnpy multiprocessing spawn grandchildren inherit
|
|
the correct database path.
|
|
|
|
Background: vnpy's run_optimization uses multiprocessing spawn, which
|
|
re-imports vnpy.trader.setting in each grandchild process. On re-import,
|
|
SETTINGS resets to defaults (database.database="database.db"), losing the
|
|
DB path configured in the parent. However, setting.py also runs
|
|
SETTINGS.update(load_json("vt_setting.json")) on import, reading from
|
|
Path.home()/.vntrader/vt_setting.json. Since sanguo-api runs as
|
|
Administrator (home=C:\\Users\\Administrator) and spawn children inherit
|
|
the same home, writing this file ensures grandchildren also load the
|
|
correct DB → load_data succeeds → calculate_result produces real
|
|
statistics instead of empty {}.
|
|
|
|
Adapter only — vnpy_v4.4.0/ source is untouched.
|
|
"""
|
|
try:
|
|
# Resolve vnpy data DB path: prefer cfg param, fall back to config file
|
|
# (same independent load as the SETTINGS setup below — cfg may be None)
|
|
if cfg and hasattr(cfg, "data_paths") and cfg.data_paths.get("vnpy_db"):
|
|
vnpy_db = cfg.data_paths["vnpy_db"]
|
|
else:
|
|
from sanguo_data.config import load_config, find_config_path
|
|
vnpy_db = load_config(find_config_path()).data_paths["vnpy_db"]
|
|
|
|
vntrader_dir = Path.home() / ".vntrader"
|
|
vntrader_dir.mkdir(parents=True, exist_ok=True)
|
|
setting = {
|
|
"database.name": "sqlite",
|
|
"database.database": vnpy_db,
|
|
}
|
|
with open(vntrader_dir / "vt_setting.json", "w", encoding="utf-8") as f:
|
|
json.dump(setting, f, ensure_ascii=False, indent=2)
|
|
except Exception as e:
|
|
logger.warning("vt_setting.json adapter failed (spawn may use default DB): %s", e)
|
|
|
|
|
|
def run_cta_optimization(
|
|
strategy_class,
|
|
symbol: str,
|
|
grid: dict,
|
|
start: str,
|
|
end: str,
|
|
cfg,
|
|
db_path: str,
|
|
max_workers: int = 2,
|
|
task_id: str = None
|
|
) -> List[BacktestResult]:
|
|
"""
|
|
Run CTA strategy parameter optimization using vnpy_ctastrategy BacktestingEngine.
|
|
|
|
Args:
|
|
strategy_class: CTA strategy class to optimize
|
|
symbol: Stock symbol (e.g., "600000")
|
|
grid: Parameter grid dict {name: (start, end, step)}
|
|
start: Backtest start date (YYYY-MM-DD format)
|
|
end: Backtest end date (YYYY-MM-DD format)
|
|
cfg: Configuration object (may contain data paths)
|
|
db_path: SQLite database path for saving results
|
|
max_workers: Maximum number of parallel optimization workers
|
|
|
|
Returns:
|
|
List[BacktestResult]: List of result objects with optimization statistics
|
|
"""
|
|
# Generate unique task ID for this optimization run (use caller-provided task_id if any)
|
|
if not task_id:
|
|
task_id = f"opt_{uuid.uuid4().hex[:8]}"
|
|
|
|
# Bug1 fix: vnpy multiprocessing spawn grandchildren re-import setting.py,
|
|
# resetting SETTINGS to defaults (database.db). Write vt_setting.json so they
|
|
# load the correct DB path. Adapter only — vnpy source untouched.
|
|
_ensure_vnpy_settings_for_spawn(cfg)
|
|
|
|
try:
|
|
# Lazy import of BacktestingEngine and OptimizationSetting
|
|
from vnpy_ctastrategy.backtesting import BacktestingEngine
|
|
from vnpy.trader.optimize import OptimizationSetting
|
|
|
|
# Build vt_symbol for A-shares
|
|
vt_symbol = f"{symbol}.{guess_exchange(symbol).value}"
|
|
|
|
# Convert date strings to datetime objects
|
|
start_dt = datetime.strptime(start, "%Y-%m-%d")
|
|
end_dt = datetime.strptime(end, "%Y-%m-%d") if end else None
|
|
|
|
# Create and configure backtesting engine
|
|
engine = BacktestingEngine()
|
|
|
|
# Set parameters with A-share specific values (same as cta_engine)
|
|
engine.set_parameters(
|
|
vt_symbol=vt_symbol,
|
|
interval="d", # Interval.DAILY.value (vnpy enum uses "d" not "1d")
|
|
start=start_dt,
|
|
end=end_dt,
|
|
rate=0.001, # Commission rate (0.1% for A-shares)
|
|
slippage=0, # No slippage for simplicity
|
|
size=1, # Contract size (1 for stocks)
|
|
pricetick=0.01, # Minimum price tick (0.01 yuan for A-shares)
|
|
capital=1_000_000, # 0 causes instant liquidation on first trade
|
|
)
|
|
|
|
# Add strategy without parameters (will be set by optimization)
|
|
engine.add_strategy(strategy_class, {})
|
|
|
|
# Configure vnpy DB → quant_trading.db (worker process; spawn isolation).
|
|
try:
|
|
from vnpy.trader.setting import SETTINGS
|
|
from sanguo_data.config import load_config, find_config_path
|
|
_dcfg = load_config(find_config_path())
|
|
SETTINGS["database.name"] = "sqlite"
|
|
SETTINGS["database.database"] = _dcfg.data_paths["vnpy_db"]
|
|
except Exception:
|
|
pass
|
|
|
|
# Load historical data
|
|
engine.load_data()
|
|
|
|
# Create optimization setting
|
|
setting = OptimizationSetting()
|
|
setting.set_target("sharpe_ratio") # Optimize for Sharpe ratio
|
|
|
|
# Add parameter ranges to optimization setting
|
|
for name, (start_val, end_val, step) in grid.items():
|
|
setting.add_parameter(name, start_val, end_val, step)
|
|
|
|
# Run optimization (headless, with parallel workers)
|
|
optimization_results = engine.run_optimization(
|
|
setting,
|
|
output=False, # Headless mode
|
|
max_workers=max_workers
|
|
)
|
|
|
|
# Parse optimization results and convert to BacktestResult objects
|
|
results = []
|
|
combos = [] # Bug2: collect valid combos for aggregate record
|
|
for item in optimization_results:
|
|
try:
|
|
# Handle both tuple format (params, target_value, statistics)
|
|
# and dict format ({params: ..., statistics: ...})
|
|
if isinstance(item, tuple) and len(item) >= 3:
|
|
params = item[0]
|
|
statistics = item[2]
|
|
elif isinstance(item, dict):
|
|
params = item.get("params", {})
|
|
statistics = item.get("statistics", {})
|
|
else:
|
|
# Unknown format, skip this result
|
|
continue
|
|
|
|
# Bug2 fix: skip items with empty statistics (vnpy returns these
|
|
# for combos that loaded 0 bars or failed). Don't persist empty records.
|
|
if not statistics or not isinstance(statistics, dict):
|
|
continue
|
|
|
|
# Create individual result for each optimization run
|
|
result = BacktestResult(
|
|
task_id=f"opt_{uuid.uuid4().hex[:8]}", # Unique ID per result
|
|
type="optimize",
|
|
status="done",
|
|
strategy=strategy_class.__name__,
|
|
symbol=symbol,
|
|
params=params,
|
|
start=start,
|
|
end=end,
|
|
statistics=statistics,
|
|
equity_curve=None, # Not available in optimization results
|
|
trades=None # Not available in optimization results
|
|
)
|
|
results.append(result)
|
|
|
|
# Save each result to database
|
|
save_result(result, db_path=db_path)
|
|
combos.append({"params": params, "statistics": statistics})
|
|
|
|
except Exception as e:
|
|
# Bug2: log parse errors but don't save empty error records
|
|
logger.warning("optimize: skipped result parse error: %s: %s",
|
|
type(e).__name__, e)
|
|
|
|
# Bug2: save aggregate record under the main task_id for persistence.
|
|
# Stores all valid combos in statistics["combos"] so results survive
|
|
# API restart and can be retrieved by the optimization-results endpoint.
|
|
if combos:
|
|
grid_summary = {name: list(rng) for name, rng in grid.items()}
|
|
aggregate = BacktestResult(
|
|
task_id=task_id,
|
|
type="optimize",
|
|
status="done",
|
|
strategy=strategy_class.__name__,
|
|
symbol=symbol,
|
|
params={
|
|
"strategy": strategy_class.__name__,
|
|
"symbol": symbol,
|
|
"grid": grid_summary,
|
|
},
|
|
start=start,
|
|
end=end,
|
|
statistics={"combos": combos},
|
|
equity_curve=None,
|
|
trades=None,
|
|
)
|
|
save_result(aggregate, db_path=db_path)
|
|
|
|
return results
|
|
|
|
except Exception as e:
|
|
# Handle any exceptions during optimization setup/execution
|
|
error_msg = f"{type(e).__name__}: {e}\n{traceback.format_exc()}"
|
|
|
|
failed_result = BacktestResult(
|
|
task_id=task_id,
|
|
type="optimize",
|
|
status="failed",
|
|
strategy=strategy_class.__name__,
|
|
symbol=symbol,
|
|
params={},
|
|
start=start,
|
|
end=end,
|
|
statistics={},
|
|
equity_curve=None,
|
|
trades=None,
|
|
error_msg=error_msg
|
|
)
|
|
|
|
# Save failed result to database
|
|
save_result(failed_result, db_path=db_path)
|
|
|
|
return [failed_result]
|
|
|
|
|
|
# Module-level reference for mocking in tests
|
|
BacktestingEngine = None # Will be set when imported inside run_cta_optimization
|
|
OptimizationSetting = None # Will be set when imported inside run_cta_optimization
|