fix(data): bs_eod 卡死根治 — per-stock commit + baostock 超时包装 + 周期 relogin
根因(py-spy dump + netstat CLOSE_WAIT 实证): baostock 服务端关长连接→CLOSE_WAIT, send_msg 静默阻塞不抛异常, socket.setdefaulttimeout 不被 baostock 自己 socket 遵守, relogin 只在 error_code≠0 救不了; 一把大事务全程持 WAL 锁阻断全库。修复: per-stock commit 去大事务 + _with_timeout 线程超时包 fetch_k 打破静默 hang + 周期 relogin 每500主动刷连接。VPS --limit 3 验证 11s 不 hang。
This commit is contained in:
@@ -17,6 +17,7 @@ import logging
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
@@ -38,6 +39,9 @@ DB = BASE / "data" / "quant_trading.db"
|
||||
VAL_DIR = BASE / "data" / "valuation_baostock"
|
||||
LOOKBACK = int(os.environ.get("LOOKBACK_DAYS", "7"))
|
||||
DAILY_LIMIT = int(os.environ.get("BS_DAILY_LIMIT", "48000"))
|
||||
# ③ 周期 relogin: 每 N 只主动 relogin, 防服务端长连接 idle 超时 → CLOSE_WAIT 静默 hang.
|
||||
# CLOSE_WAIT 是静默阻塞不抛异常, 被动 relogin (error_code != 0 触发) 救不了, 必须周期主动刷连接.
|
||||
RELOGIN_EVERY = int(os.environ.get("BS_RELOGIN_EVERY", "500"))
|
||||
BS_INTERVAL = 0.3
|
||||
QUERY_COUNT = 0
|
||||
EXC_MAP = {"sh": "SSE", "sz": "SZSE"}
|
||||
@@ -114,6 +118,57 @@ def fetch_k(bs_code, fields, freq, start, end):
|
||||
return rows
|
||||
|
||||
|
||||
# ======================== 超时包装 ========================
|
||||
# 根因: socket.setdefaulttimeout(30) 对 baostock 客户端的 rs.next() / recv 不可靠遵守,
|
||||
# 服务端 hiccup 会无限阻塞主循环. 用 daemon 工作线程 + join(timeout) 强制上限.
|
||||
#
|
||||
# 线程方案 vs subprocess 隔离: 选线程.
|
||||
# 理由: (1) baostock 模块级 singleton, 主线程 join 等子线程 → 单线程串行调用安全;
|
||||
# (2) 超时后子线程 daemon 化泄漏, 后续 relogin 的 logout 关旧 socket → 旧线程
|
||||
# recv 报错自死, 新 login 走新 socket 不受污染 (relogin retry 兜底恢复);
|
||||
# (3) subprocess 方案需重新 login (~1s) + 序列化 rows 复杂, 不值;
|
||||
# (4) 与 akshare_static_download.call_ak_with_timeout 同范式.
|
||||
|
||||
def _with_timeout(fn, args=(), kwargs=None, timeout=60):
|
||||
"""daemon 线程跑 fn(*args, **kwargs), timeout 秒未完成 raise TimeoutError.
|
||||
|
||||
超时后工作线程泄漏 (daemon=True, 进程退出时强杀); 主线程立即返回让上层 relogin.
|
||||
子线程异常透传给主线程 (BaseException 也捕获, 避免 daemon 吞 KeyboardInterrupt).
|
||||
"""
|
||||
if kwargs is None:
|
||||
kwargs = {}
|
||||
box = {"val": None, "exc": None}
|
||||
|
||||
def worker():
|
||||
try:
|
||||
box["val"] = fn(*args, **kwargs)
|
||||
except BaseException as e: # noqa: BLE001 - 透传所有异常含 KeyboardInterrupt
|
||||
box["exc"] = e
|
||||
|
||||
t = threading.Thread(target=worker, daemon=True)
|
||||
t.start()
|
||||
t.join(timeout)
|
||||
if t.is_alive():
|
||||
raise TimeoutError(f"{getattr(fn, '__name__', repr(fn))} 超过 {timeout}s")
|
||||
if box["exc"] is not None:
|
||||
raise box["exc"]
|
||||
return box["val"]
|
||||
|
||||
|
||||
def fetch_k_with_timeout(bs_code, fields, freq, start, end, timeout=60):
|
||||
"""fetch_k + 超时保护 (默认 60s; baostock hiccup 不再无限阻塞)."""
|
||||
return _with_timeout(
|
||||
fetch_k,
|
||||
args=(bs_code, fields, freq, start, end),
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def fetch_all_stocks_with_timeout(timeout=120):
|
||||
"""fetch_all_stocks + 超时保护 (全 A 列表一次性返回, 给 120s)."""
|
||||
return _with_timeout(fetch_all_stocks, timeout=timeout)
|
||||
|
||||
|
||||
def upsert_daily(conn, code, prefix, rows):
|
||||
"""日线 rows -> dbbardata('d') + valuation_baostock 当年 parquet 追加。"""
|
||||
if not rows:
|
||||
@@ -191,6 +246,27 @@ def upsert_15m(conn, code, prefix, rows):
|
||||
return len(db)
|
||||
|
||||
|
||||
def _process_one_stock(conn, code, prefix, args, start, end):
|
||||
"""单只股票: fetch_k + upsert, 在 with conn 短事务里执行 (大事务根治).
|
||||
|
||||
每只股票一个事务 — hang/kill/异常最多丢 1 只, 已 commit 的其他股不受影响.
|
||||
fetch 也包在事务里 (task 要求: "fetch_k + upsert 包在自己事务里");
|
||||
fetch 用 fetch_k_with_timeout 保护, 网络挂最多锁 timeout 秒.
|
||||
成功返 (n1_daily, n2_15m); 异常时 with conn 自动 ROLLBACK 该股, 异常上抛.
|
||||
"""
|
||||
bs_code = f"{prefix}.{code}"
|
||||
n1 = 0
|
||||
n2 = 0
|
||||
with conn: # 显式短事务: 成功 commit / 异常 rollback (per-stock 原子)
|
||||
if not args.no_daily:
|
||||
d_rows = fetch_k_with_timeout(bs_code, DAILY_FIELDS, "d", start, end)
|
||||
n1 = upsert_daily(conn, code, prefix, d_rows)
|
||||
if not args.no_15m:
|
||||
m_rows = fetch_k_with_timeout(bs_code, M15_FIELDS, "15", start, end)
|
||||
n2 = upsert_15m(conn, code, prefix, m_rows)
|
||||
return n1, n2
|
||||
|
||||
|
||||
def main():
|
||||
global QUERY_COUNT
|
||||
ap = argparse.ArgumentParser()
|
||||
@@ -210,7 +286,7 @@ def main():
|
||||
sys.exit(2)
|
||||
|
||||
try:
|
||||
stocks = fetch_all_stocks()
|
||||
stocks = fetch_all_stocks_with_timeout()
|
||||
except Exception as e:
|
||||
log.error("[FATAL] fetch_all: %s", e)
|
||||
sys.exit(1)
|
||||
@@ -227,45 +303,39 @@ def main():
|
||||
stats = {"ok": 0, "empty": 0, "failed": 0, "db_rows": 0}
|
||||
limit_reached = False
|
||||
t0 = time.time()
|
||||
conn.execute("BEGIN")
|
||||
# 大事务根治: 不再 BEGIN/COMMIT 包全程. 每只股票 with conn 短事务独立提交,
|
||||
# hang/kill/崩溃最多丢 1 只 (per-stock 隔离), 已 commit 的进度不丢.
|
||||
try:
|
||||
for i, (code, prefix) in enumerate(stocks):
|
||||
if QUERY_COUNT >= DAILY_LIMIT:
|
||||
log.warning("query %d 达防线 %d, graceful stop", QUERY_COUNT, DAILY_LIMIT)
|
||||
limit_reached = True
|
||||
break
|
||||
bs_code = f"{prefix}.{code}"
|
||||
try:
|
||||
n1 = 0
|
||||
if not args.no_daily:
|
||||
d_rows = fetch_k(bs_code, DAILY_FIELDS, "d", start, end)
|
||||
n1 = upsert_daily(conn, code, prefix, d_rows)
|
||||
n2 = 0
|
||||
if not args.no_15m:
|
||||
m_rows = fetch_k(bs_code, M15_FIELDS, "15", start, end)
|
||||
n2 = upsert_15m(conn, code, prefix, m_rows)
|
||||
n1, n2 = _process_one_stock(conn, code, prefix, args, start, end)
|
||||
stats["db_rows"] += n1 + n2
|
||||
if n1 + n2:
|
||||
stats["ok"] += 1
|
||||
else:
|
||||
stats["empty"] += 1
|
||||
except Exception as e:
|
||||
# _process_one_stock 异常已 rollback 该股, 其他股已 commit 不受影响
|
||||
stats["failed"] += 1
|
||||
if stats["failed"] <= 5:
|
||||
if stats["failed"] <= 5 or stats["failed"] % 100 == 0:
|
||||
log.warning("%s err: %s", code, e)
|
||||
if not relogin():
|
||||
log.error("%s relogin 失败, 跳过", code)
|
||||
if (i + 1) % 500 == 0:
|
||||
if (i + 1) % RELOGIN_EVERY == 0:
|
||||
log.info("进度 %d/%d ok=%d empty=%d failed=%d q=%d (%.0fs)",
|
||||
i + 1, len(stocks), stats["ok"], stats["empty"],
|
||||
stats["failed"], QUERY_COUNT, time.time() - t0)
|
||||
# ③ 主动 relogin (双保险之治本): 服务端长连接 idle 超时 → CLOSE_WAIT 静默 hang,
|
||||
# 被动 relogin 不触发 (不抛异常), 必须周期主动 logout+login 刷新连接.
|
||||
# ② _with_timeout 是治标兜底, 真挂了能打破; 这里治本避免走到那一步.
|
||||
if not relogin():
|
||||
log.warning("周期 relogin 失败, 继续跑 (下次 fetch 失败时被动 relogin 兜底)")
|
||||
if i < len(stocks) - 1:
|
||||
time.sleep(BS_INTERVAL)
|
||||
conn.execute("COMMIT")
|
||||
except Exception as e:
|
||||
conn.execute("ROLLBACK")
|
||||
log.error("[FATAL] rollback: %s", e)
|
||||
sys.exit(1)
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user