diff --git a/scripts/data_platform/bs_eod.py b/scripts/data_platform/bs_eod.py index 66fcc23..00a194d 100644 --- a/scripts/data_platform/bs_eod.py +++ b/scripts/data_platform/bs_eod.py @@ -154,6 +154,20 @@ def upsert_daily(conn, code, prefix, rows): return len(db) +def _build_15m_dt(date_series, time_series): + """baostock 15min datetime 拼接: date="2026-07-21"(带 -) + time="20260721094500000"(17 位)。 + + 从 17 位 time 第 8-12 位提取 HHMM, date 直连(带 -)。 + 产出 'YYYY-MM-DD HH:MM:00' (符合 dbbardata GLOB 模式, 不被清理误删)。 + + bug 根因(2026-07-25): 原代码假设 date 纯数字 + time[:6] 取年月, + 但 baostock 实测 date 带 -, time[:6]=YYYYMM, 产乱 datetime 致 15min 全市场停 7-17。 + """ + _t = time_series.astype(str) + return (date_series.astype(str) + " " + + _t.str.slice(8, 10) + ":" + _t.str.slice(10, 12) + ":00") + + def upsert_15m(conn, code, prefix, rows): if not rows: return 0 @@ -161,8 +175,7 @@ def upsert_15m(conn, code, prefix, rows): for c in ["open", "high", "low", "close", "volume", "amount"]: df[c] = pd.to_numeric(df[c], errors="coerce") exc = EXC_MAP[prefix] - dt_col = (df["date"].astype(str) + " " + df["time"].astype(str).str.slice(0, 6) - ).apply(lambda s: f"{s[0:4]}-{s[4:6]}-{s[6:8]} {s[8:10]}:{s[10:12]}:00") + dt_col = _build_15m_dt(df["date"], df["time"]) db = pd.DataFrame({ "symbol": code, "exchange": exc, "datetime": dt_col, "interval": "15m", "volume": df["volume"], "turnover": df["amount"], @@ -183,6 +196,8 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument("--limit", type=int, default=0) ap.add_argument("--no-15m", action="store_true") + ap.add_argument("--no-daily", action="store_true", + help="跳日线, 只跑 15min(用于 15min 重灌快)") args = ap.parse_args() today = dt.date.today() @@ -221,8 +236,10 @@ def main(): break bs_code = f"{prefix}.{code}" try: - d_rows = fetch_k(bs_code, DAILY_FIELDS, "d", start, end) - n1 = upsert_daily(conn, code, prefix, d_rows) + n1 = 0 + if not args.no_daily: + d_rows = fetch_k(bs_code, DAILY_FIELDS, "d", start, end) + n1 = upsert_daily(conn, code, prefix, d_rows) n2 = 0 if not args.no_15m: m_rows = fetch_k(bs_code, M15_FIELDS, "15", start, end) diff --git a/tests/data_platform/conftest.py b/tests/data_platform/conftest.py new file mode 100644 index 0000000..25a277c --- /dev/null +++ b/tests/data_platform/conftest.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +"""pytest 路径修补: 让 scripts/data_platform/* 的 sibling import 在测试下可用。 + +VPS 上 bs_eod.py 作为脚本跑(scripts/data_platform 在 sys.path[0]), +`from dbbardata_utils import ...` 可用。pytest 以包导入 `scripts.data_platform.bs_eod` +时 sibling import 失败 — 此处把 script dir 加到 sys.path 兼容两种运行模式。 +""" +import sys +from pathlib import Path + +_SCRIPT_DIR = Path(__file__).resolve().parents[2] / "scripts" / "data_platform" +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) diff --git a/tests/data_platform/test_bs_eod_15m_dt.py b/tests/data_platform/test_bs_eod_15m_dt.py new file mode 100644 index 0000000..7b8ec36 --- /dev/null +++ b/tests/data_platform/test_bs_eod_15m_dt.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +"""TDD for bs_eod._build_15m_dt (15min datetime 格式 bug 根治). + +背景: + baostock 15min 实测返回 date="2026-07-21"(带 -), time="20260721094500000" + (17 位 YYYYMMDDHHMMSSmmm)。原代码假设 date 纯数字 + time[:6] 取年月, + 产出乱 datetime 致 dbbardata 15min 全市场停 2026-07-17 (8 天没正确累积)。 + +修复策略: 提纯函数 _build_15m_dt(date_series, time_series) 单测。 +""" +import pandas as pd +import pytest + +from scripts.data_platform.bs_eod import _build_15m_dt + + +def _series(date_str: str, time_str: str) -> tuple[pd.Series, pd.Series]: + return pd.Series([date_str]), pd.Series([time_str]) + + +@pytest.mark.parametrize("date_str,time_str,expected", [ + # 早盘 09:45 + ("2026-07-21", "20260721094500000", "2026-07-21 09:45:00"), + # 午盘 14:30 + ("2026-07-21", "20260721143000000", "2026-07-21 14:30:00"), + # 收盘 15:00 + ("2026-07-21", "20260721150000000", "2026-07-21 15:00:00"), + # 午盘 13:00 + ("2026-07-21", "20260721130000000", "2026-07-21 13:00:00"), + # 跨日 2025-12-31 14:45 + ("2025-12-31", "20251231144500000", "2025-12-31 14:45:00"), +]) +def test_build_15m_dt_single_row(date_str, time_str, expected): + """单行: 从 17 位 time 第 8-12 位提取 HHMM, date 直连(带 -).""" + d, t = _series(date_str, time_str) + out = _build_15m_dt(d, t) + assert list(out) == [expected] + + +def test_build_15m_dt_multi_row_mixed_times(): + """多行混合: 同日不同时分, 产出对应 HH:MM:00.""" + dates = pd.Series(["2026-07-21"] * 4) + times = pd.Series([ + "20260721094500000", + "20260721100000000", + "20260721143000000", + "20260721150000000", + ]) + out = list(_build_15m_dt(dates, times)) + assert out == [ + "2026-07-21 09:45:00", + "2026-07-21 10:00:00", + "2026-07-21 14:30:00", + "2026-07-21 15:00:00", + ] + + +def test_build_15m_dt_multi_row_different_dates(): + """多行不同日期: 跨日场景.""" + dates = pd.Series(["2026-07-21", "2026-07-22", "2026-07-23"]) + times = pd.Series([ + "20260721150000000", + "20260722150000000", + "20260723150000000", + ]) + out = list(_build_15m_dt(dates, times)) + assert out == [ + "2026-07-21 15:00:00", + "2026-07-22 15:00:00", + "2026-07-23 15:00:00", + ] + + +def test_build_15m_dt_format_glob_safe(): + """产出格式必须符合 'YYYY-MM-DD HH:MM:SS' GLOB 模式 + (用于 VPS 清理乱 datetime 时不会误删正常行).""" + d, t = _series("2026-07-21", "20260721094500000") + out = _build_15m_dt(d, t) + val = out.iloc[0] + # GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]' + assert len(val) == 19 + assert val[4] == "-" and val[7] == "-" and val[10] == " " + assert val[13] == ":" and val[16] == ":" + # 每位均数字 + digits = val.replace("-", "").replace(":", "").replace(" ", "") + assert digits.isdigit() and len(digits) == 14