claude_dev
6ed71ed8cf
根因: vnpy run_optimization 孙进程 spawn re-import setting.py 重置 DB → load 0根 → 空 statistics;连带 task_id 不一致 + statistics 含 date/numpy 序列化失败 + 前端单页轮询 180s 超时。 修复(不动 vnpy 源码): - cta_optimizer: vt_setting.json 适配(vnpy 原生机制 setting.py:43 load_json,孙进程拿到正确 DB) + task_id 贯通 + 主聚合持久化(combos) + 过滤空 statistics 记录 - result_store: _json_default(date→isoformat + numpy→原生),save_result 两处 json.dumps 加 default - routes: optimization-results 改读 DB 主聚合, fallback 内存(重启不丢) - runner: _opt_worker 加 task_id 参数 + submit_optimize 传递(对齐 _cta_worker) - 前端: Optimize 提交后跳 progress;Progress 用 opt_ 前缀判断跳 optimize-result;新建 OptimizeResult 参数表格页;router 加路由 验证: 端到端 optimization-results http=200 + 4组合非空 statistics + 浏览器提交→跳 optimize-result 页表格渲染。
237 lines
7.5 KiB
Python
237 lines
7.5 KiB
Python
"""Backtest result storage using SQLite + parquet files."""
|
|
import sqlite3
|
|
import json
|
|
import logging
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
import pandas as pd
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _json_default(obj):
|
|
"""json.dumps default: vnpy calculate_result 的 statistics 含 numpy.int64/float64,
|
|
标准 json 不能序列化（optimize 的统计直接来自 vnpy，未像 cta 那样被 compute_metrics
|
|
覆盖成 python float）。用 duck typing 把 numpy 标量/数组转原生，不引入 numpy 硬依赖。"""
|
|
import datetime as _dt
|
|
if isinstance(obj, (_dt.date, _dt.datetime)):
|
|
return obj.isoformat()
|
|
if hasattr(obj, "item") and callable(getattr(obj, "item")):
|
|
try:
|
|
return obj.item()
|
|
except Exception:
|
|
pass
|
|
if hasattr(obj, "tolist") and callable(getattr(obj, "tolist")):
|
|
try:
|
|
return obj.tolist()
|
|
except Exception:
|
|
pass
|
|
raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable")
|
|
|
|
|
|
@dataclass
|
|
class BacktestResult:
|
|
"""Backtest result data structure."""
|
|
task_id: str
|
|
type: str
|
|
status: str
|
|
strategy: str
|
|
symbol: str
|
|
params: dict
|
|
start: str
|
|
end: str
|
|
statistics: dict
|
|
equity_curve: Optional[pd.DataFrame] = None
|
|
trades: Optional[pd.DataFrame] = None
|
|
error_msg: Optional[str] = None
|
|
id: Optional[int] = None
|
|
|
|
|
|
# SQLite schema for backtest stats
|
|
_SCHEMA = """CREATE TABLE IF NOT EXISTS backtest_stats (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
task_id TEXT,
|
|
type TEXT,
|
|
status TEXT,
|
|
strategy TEXT,
|
|
symbol TEXT,
|
|
params TEXT,
|
|
start TEXT,
|
|
end TEXT,
|
|
statistics TEXT,
|
|
equity_path TEXT,
|
|
trades_path TEXT,
|
|
error_msg TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);"""
|
|
|
|
|
|
def _connect(db_path: str) -> sqlite3.Connection:
|
|
"""Create database connection and initialize schema."""
|
|
conn = sqlite3.connect(db_path)
|
|
conn.executescript(_SCHEMA)
|
|
return conn
|
|
|
|
|
|
def save_result(result: BacktestResult, db_path: str, file_dir: Optional[str] = None) -> int:
|
|
"""
|
|
Save backtest result to database and optionally save DataFrame fields to parquet files.
|
|
|
|
Args:
|
|
result: BacktestResult object to save
|
|
db_path: Path to SQLite database file
|
|
file_dir: Optional directory to save parquet files (equity_curve and trades)
|
|
|
|
Returns:
|
|
int: The ID of the inserted record
|
|
"""
|
|
conn = _connect(db_path)
|
|
try:
|
|
equity_path = None
|
|
trades_path = None
|
|
|
|
# Save DataFrames to parquet if file_dir is provided
|
|
if file_dir:
|
|
fdir = Path(file_dir)
|
|
fdir.mkdir(parents=True, exist_ok=True)
|
|
|
|
if result.equity_curve is not None and not result.equity_curve.empty:
|
|
equity_path = str(fdir / f"{result.task_id}_equity.json")
|
|
result.equity_curve.to_json(equity_path, orient="records", date_format="iso", force_ascii=False)
|
|
|
|
if result.trades is not None and not result.trades.empty:
|
|
trades_path = str(fdir / f"{result.task_id}_trades.json")
|
|
result.trades.to_json(trades_path, orient="records", date_format="iso", force_ascii=False)
|
|
|
|
# Insert record into database
|
|
cur = conn.execute(
|
|
"""INSERT INTO backtest_stats
|
|
(task_id, type, status, strategy, symbol, params, start, end,
|
|
statistics, equity_path, trades_path, error_msg)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
(
|
|
result.task_id,
|
|
result.type,
|
|
result.status,
|
|
result.strategy,
|
|
result.symbol,
|
|
json.dumps(result.params, default=_json_default),
|
|
result.start,
|
|
result.end,
|
|
json.dumps(result.statistics, default=_json_default),
|
|
equity_path,
|
|
trades_path,
|
|
result.error_msg
|
|
)
|
|
)
|
|
conn.commit()
|
|
result.id = cur.lastrowid
|
|
return cur.lastrowid
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def _safe_read_json(path: Optional[str]) -> Optional[pd.DataFrame]:
|
|
"""Read a JSON equity/trades file; return None if missing or unreadable.
|
|
|
|
Historical records may reference paths from a previous host (e.g. NAS
|
|
absolute paths after migration to VPS). Swallow those failures so the
|
|
record stays listable instead of crashing list_results.
|
|
"""
|
|
if not path:
|
|
return None
|
|
try:
|
|
return pd.read_json(path, orient="records")
|
|
except (FileNotFoundError, ValueError, OSError) as e:
|
|
logger.warning("result_store: skipping unreadable JSON file %s: %s", path, e)
|
|
return None
|
|
|
|
|
|
def load_result(rid: int, db_path: str) -> BacktestResult:
|
|
"""
|
|
Load backtest result by ID from database.
|
|
|
|
Args:
|
|
rid: Result ID to load
|
|
db_path: Path to SQLite database file
|
|
|
|
Returns:
|
|
BacktestResult: Loaded result object
|
|
|
|
Raises:
|
|
KeyError: If result ID not found
|
|
"""
|
|
conn = _connect(db_path)
|
|
try:
|
|
row = conn.execute("SELECT * FROM backtest_stats WHERE id=?", (rid,)).fetchone()
|
|
if not row:
|
|
raise KeyError(f"result {rid} not found")
|
|
|
|
# Get column names from table description
|
|
cols = [d[0] for d in conn.execute("SELECT * FROM backtest_stats LIMIT 0").description]
|
|
d = dict(zip(cols, row))
|
|
|
|
# Load JSON files if paths exist (equity_curve/trades persisted as JSON).
|
|
# Tolerate stale paths (e.g. NAS absolute paths left after VPS migration):
|
|
# missing/unreadable file -> None, so record still appears in list_results.
|
|
equity = _safe_read_json(d.get("equity_path"))
|
|
trades = _safe_read_json(d.get("trades_path"))
|
|
|
|
return BacktestResult(
|
|
task_id=d["task_id"],
|
|
type=d["type"],
|
|
status=d["status"],
|
|
strategy=d["strategy"],
|
|
symbol=d["symbol"],
|
|
params=json.loads(d["params"]),
|
|
start=d["start"],
|
|
end=d["end"],
|
|
statistics=json.loads(d["statistics"]) if d["statistics"] else {},
|
|
equity_curve=equity,
|
|
trades=trades,
|
|
error_msg=d.get("error_msg")
|
|
)
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def load_result_by_task_id(task_id: str, db_path: str) -> BacktestResult | None:
|
|
"""Load the most recent result for a task_id (historical lookup after restart)."""
|
|
conn = _connect(db_path)
|
|
try:
|
|
row = conn.execute(
|
|
"SELECT id FROM backtest_stats WHERE task_id=? ORDER BY id DESC LIMIT 1",
|
|
(task_id,),
|
|
).fetchone()
|
|
if not row:
|
|
return None
|
|
return load_result(row[0], db_path)
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def list_results(type_filter: Optional[str] = None, db_path: str = "") -> list[BacktestResult]:
|
|
"""
|
|
List all backtest results, optionally filtered by type.
|
|
|
|
Args:
|
|
type_filter: Optional filter for result type (e.g., 'cta', 'factor')
|
|
db_path: Path to SQLite database file
|
|
|
|
Returns:
|
|
list[BacktestResult]: List of loaded result objects
|
|
"""
|
|
conn = _connect(db_path)
|
|
try:
|
|
query = "SELECT id FROM backtest_stats"
|
|
args = ()
|
|
|
|
if type_filter:
|
|
query = query + " WHERE type=?"
|
|
args = (type_filter,)
|
|
|
|
return [load_result(row[0], db_path) for row in conn.execute(query, args).fetchall()]
|
|
finally:
|
|
conn.close()
|