feat(data): NAS独立5分钟线备份——bs_5m_eod新脚本(NAS专属,VPS永不跑)——用户拍板2026-08-20:NAS既有数据全是VPS同步镜像(nas_sync零接触),5m是NAS唯一自主从baostock下载数据,落独立库BS_5M_DB(缺省/volume1/stock/sanguo_5m/dbbardata_5m.db容器内外同路径),绝不写同步目标data_backup/quant_trading.db;①首次自建schema+唯一索引(REPLACE去重)②缺省每日增量LOOKBACK=7(DSM任务计划~19:05);--full一次性回灌2020-01-03+(baostock分钟固定起点非滚动;全区间每股恰1次调用=~5.5k query一趟≈3.6亿行,数小时级,per-stock短事务断点续跑幂等;FULL_TIMEOUT=300s防大payload误杀)③配额:NAS出口IP独立核算48k/天与VPS互不相干,DAILY_LIMIT守卫保留④单实例锁bs_5m.lock(pid活性检测,死锁自动覆盖/PermissionError按在跑)防DSM定时与--full长跑撞车=同IP双baostock连接红线⑤复用bs_eod四件套+datetime拼接/数值口径与15m完全同款;+13测试(schema/REPLACE不双行/per-stock原子/full vs daily窗口/锁三态/main四条);data_platform 153绿 [nas]

2026-08-20 09:58:00 +08:00
parent d0a538ced3
commit a6f924e950
2 changed files with 414 additions and 0 deletions
@@ -0,0 +1,189 @@
+# -*- coding: utf-8 -*-
+"""TDD for bs_5m_eod.py — NAS 独立 5 分钟线备份 (2026-08-20 用户拍板).
+
+定位: NAS 自己从 baostock 下载 5m, 不推 VPS; NAS 其余数据全是 VPS 同步镜像,
+本脚本是 NAS 唯一自主数据 —— 落独立库 BS_5M_DB(缺省 /volume1/stock/sanguo_5m/),
+绝不碰同步目录 /volume1/stock/sanguo_vnpy_v2/data_backup/quant_trading.db。
+
+配额: NAS 出口 IP 独立核算 48k/天, 一趟仅 ~5.5k query(每股恰好 1 次调用,
+--full 全区间也一样); VPS 侧 bs_eod/bs_fund 用 VPS 自己的 IP 配额, 互不相干。
+"""
+import datetime as dt
+import os
+import sqlite3
+import sys
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+if "baostock" not in sys.modules:
+    sys.modules["baostock"] = MagicMock()
+
+from scripts.data_platform import bs_5m_eod as b5  # noqa: E402
+
+
+# ---------- Fixtures ----------
+
+@pytest.fixture(autouse=True)
+def _fast(monkeypatch):
+    monkeypatch.setattr(b5, "BS_INTERVAL", 0.0)
+
+
+@pytest.fixture
+def tmp_db_conn(tmp_path, monkeypatch):
+    """重定向 DB 到 tmp_path, 返已建 schema 的连接(测完关)."""
+    db = tmp_path / "dbbardata_5m.db"
+    monkeypatch.setattr(b5, "DB", db)
+    conn = sqlite3.connect(str(db))
+    b5.ensure_schema(conn)
+    yield conn
+    conn.close()
+
+
+def _m5_rows(bs_code, n=1, hhmm="0935"):
+    """模拟 baostock 5min row (9 列 M5_FIELDS, time 17 位)."""
+    out = []
+    for i in range(n):
+        out.append(["2026-07-01", f"20260701{hhmm}00000", bs_code,
+                    "10", "11", "9", "10.5", "1000", "10000"])
+    return out
+
+
+# ---------- schema ----------
+
+def test_ensure_schema_creates_table_and_unique_index(tmp_path):
+    db = tmp_path / "x.db"
+    conn = sqlite3.connect(str(db))
+    b5.ensure_schema(conn)
+    names = {r[0] for r in conn.execute(
+        "SELECT name FROM sqlite_master WHERE tbl_name='dbbardata'")}
+    assert "dbbardata" in names
+    assert "dbbardata_symbol_exchange_interval_datetime" in names  # REPLACE 去重前提
+    conn.close()
+
+
+# ---------- upsert_5m ----------
+
+def test_upsert_5m_writes_dbbardata(tmp_db_conn):
+    n = b5.upsert_5m(tmp_db_conn, "000001", "sz", _m5_rows("sz.000001"))
+    tmp_db_conn.commit()
+    assert n == 1
+    row = tmp_db_conn.execute(
+        "SELECT symbol, exchange, datetime, interval, close_price FROM dbbardata"
+    ).fetchone()
+    assert row[0] == "000001"
+    assert row[1] == "SZSE"
+    assert row[2] == "2026-07-01 09:35:00"  # 17 位 time 第 8-12 位取 HHMM
+    assert row[3] == "5m"
+    assert abs(row[4] - 10.5) < 1e-6
+
+
+def test_upsert_5m_replaces_same_pk_no_dup(tmp_db_conn):
+    b5.upsert_5m(tmp_db_conn, "000001", "sz", _m5_rows("sz.000001"))
+    b5.upsert_5m(tmp_db_conn, "000001", "sz", _m5_rows("sz.000001"))
+    tmp_db_conn.commit()
+    assert tmp_db_conn.execute("SELECT COUNT(*) FROM dbbardata").fetchone()[0] == 1
+
+
+def test_upsert_5m_empty_rows_no_op(tmp_db_conn):
+    assert b5.upsert_5m(tmp_db_conn, "000001", "sz", []) == 0
+    assert tmp_db_conn.execute("SELECT COUNT(*) FROM dbbardata").fetchone()[0] == 0
+
+
+def test_process_one_per_stock_atomic(tmp_db_conn, monkeypatch):
+    """单股独立事务: fetch 抛错 → 整股 rollback 不留半行(per-stock 原子, house style)."""
+    with patch.object(b5, "fetch_k_with_timeout",
+                      side_effect=RuntimeError("baostock hiccup")):
+        with pytest.raises(RuntimeError):
+            b5._process_one(tmp_db_conn, "000001", "sz", "2026-08-01",
+                            "2026-08-19", timeout=60)
+    assert tmp_db_conn.execute("SELECT COUNT(*) FROM dbbardata").fetchone()[0] == 0
+
+
+def test_process_one_success_writes(tmp_db_conn):
+    with patch.object(b5, "fetch_k_with_timeout",
+                      side_effect=lambda *a, **k: _m5_rows("sz.000001")):
+        n = b5._process_one(tmp_db_conn, "000001", "sz", "2026-08-01",
+                            "2026-08-19", timeout=60)
+    assert n == 1
+
+
+# ---------- 窗口 ----------
+
+def test_calc_window_full_vs_daily():
+    today = dt.date(2026, 8, 20)
+    assert b5.calc_window(today, full=True) == ("2020-01-03", "2026-08-20")
+    s, e = b5.calc_window(today, full=False)
+    assert s == "2026-08-13" and e == "2026-08-20"  # LOOKBACK=7
+
+
+# ---------- 单实例锁(防 DSM 定时与全量回灌撞车) ----------
+
+def test_lock_acquire_and_second_refused(tmp_path, monkeypatch):
+    monkeypatch.setattr(b5, "DB", tmp_path / "dbbardata_5m.db")
+    assert b5._acquire_lock() is True       # 活进程(自己)持锁
+    assert b5._acquire_lock() is False      # 第二实例拒绝
+    b5._release_lock()
+    assert b5._acquire_lock() is True       # 释放后可再取
+
+
+def test_lock_stale_pid_reclaimed(tmp_path, monkeypatch):
+    monkeypatch.setattr(b5, "DB", tmp_path / "dbbardata_5m.db")
+    lock = tmp_path / "bs_5m.lock"
+    lock.write_text("999999", encoding="utf-8")  # 死 pid(无此进程)
+    assert b5._acquire_lock() is True
+    b5._release_lock()
+
+
+# ---------- main 集成 ----------
+
+@pytest.fixture
+def main_env(tmp_path, monkeypatch):
+    monkeypatch.setattr(b5, "DB", tmp_path / "dbbardata_5m.db")
+    monkeypatch.setattr(sys, "argv", ["bs_5m_eod.py"])
+    monkeypatch.setattr(b5, "login_with_retry", MagicMock(return_value=True))
+    monkeypatch.setattr(b5, "fetch_all_stocks_with_timeout",
+                        MagicMock(return_value=[("600001", "sh"), ("000002", "sz")]))
+    return tmp_path
+
+
+def test_main_runs_and_exits_0(main_env):
+    with patch.object(b5, "_process_one", return_value=48) as m_proc:
+        with pytest.raises(SystemExit) as e:
+            b5.main()
+    assert e.value.code == 0
+    assert m_proc.call_count == 2
+    # 数据库文件+schema 已就位
+    conn = sqlite3.connect(str(b5.DB))
+    assert conn.execute(
+        "SELECT COUNT(*) FROM sqlite_master "
+        "WHERE tbl_name='dbbardata' AND type='table'"
+    ).fetchone()[0] == 1
+    conn.close()
+    # 正常退出释放锁
+    assert not (main_env / "bs_5m.lock").exists()
+
+
+def test_main_exit2_when_login_fails(main_env, monkeypatch):
+    monkeypatch.setattr(b5, "login_with_retry", MagicMock(return_value=False))
+    with pytest.raises(SystemExit) as e:
+        b5.main()
+    assert e.value.code == 2
+
+
+def test_main_exit3_when_limit_reached(main_env, monkeypatch):
+    monkeypatch.setattr(b5, "DAILY_LIMIT", 0)  # 每股恰 1 query, 0=立即达限
+    with patch.object(b5, "_process_one",
+                      side_effect=AssertionError("不应跑股")):
+        with pytest.raises(SystemExit) as e:
+            b5.main()
+    assert e.value.code == 3
+
+
+def test_main_second_instance_skips(main_env, monkeypatch):
+    monkeypatch.setattr(b5, "_acquire_lock", MagicMock(return_value=False))
+    monkeypatch.setattr(b5, "login_with_retry",
+                        MagicMock(side_effect=AssertionError("不应登录")))
+    with pytest.raises(SystemExit) as e:
+        b5.main()
+    assert e.value.code == 0