From 8b2693423a5d71d42e309e87205484318c5b5cf2 Mon Sep 17 00:00:00 2001 From: claude_dev Date: Mon, 27 Jul 2026 20:56:16 +0800 Subject: [PATCH] =?UTF-8?q?feat(data):=20xt=5Feod=20=E6=89=A9=E5=B1=95?= =?UTF-8?q?=E8=A1=A5=E5=85=A8=E5=8C=97=E4=BA=A4=E6=89=80=20920xxx=20?= =?UTF-8?q?=E6=97=A5=E7=BA=BF(exc=5Fof+universe+import=E5=AE=B9=E9=94=99,?= =?UTF-8?q?=E5=A4=8D=E7=94=A8=C3=97100=20normalize)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - exc_of: 920 -> BJSE (3 位前缀优先于 2 位 SSE/SZSE 判断, 防 sym[:2]='92' 落 SZSE) - universe: 沪深ETF/基金 ∪ 北交所920xxx (从 constituent_unified 932000 取, baostock 不覆盖) - import xtquant 容错 (mac xd=None 可单测 exc_of, main() 开头 return 2) - --full-bj: 北交所 backfill start=20240101 (默认 LOOKBACK=30 与 ETF 同窗) - 复用 normalize_daily_dt + ×100 volume 口径, 不动已有 ETF 写入路径 Tests: tests/data_platform/test_xt_eod_bj.py 14 cases RED -> GREEN (3 位前缀 critical + ETF/沪深/深市 覆盖) --- scripts/data_platform/xt_eod.py | 76 ++++++++++++++++++++------- tests/data_platform/test_xt_eod_bj.py | 46 ++++++++++++++++ 2 files changed, 103 insertions(+), 19 deletions(-) create mode 100644 tests/data_platform/test_xt_eod_bj.py diff --git a/scripts/data_platform/xt_eod.py b/scripts/data_platform/xt_eod.py index 78baded..8291c4e 100644 --- a/scripts/data_platform/xt_eod.py +++ b/scripts/data_platform/xt_eod.py @@ -1,24 +1,29 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -"""xt_eod.py — sanguo-xt-eod (方案A schtask 18:40): xtata ETF/基金 EOD 增量 -> dbbardata('d')。 +"""xt_eod.py — sanguo-xt-eod (方案A schtask 18:40): xtata ETF/基金/北交所 EOD 增量 -> dbbardata('d')。 -baostock 只取 type=1 股票, 不覆盖 ETF/基金 -> xtata 独占 (spec §14)。 -个股日线由 bs_eod (baostock) 灌, 此处不碰个股, 避免 dbbardata 两源冲突。 +baostock 只取 type=1 股票, 不覆盖 ETF/基金/北交所个股 -> xtata 独占 (spec §14)。 +北交所个股 baostock 不覆盖, 此处补; 沪深个股仍由 bs_eod 灌, 避免 dbbardata 两源冲突。 -- universe = 沪深ETF ∪ 沪深基金 +- universe = 沪深ETF ∪ 沪深基金 ∪ 北交所920xxx (中证2000 成份股, baostock 不覆盖, xtata 独占) - download_history_data2 批量 paced -> 本地缓存 - get_market_data_ex raw(dividend_type=none) -> dbbardata('d') INSERT OR REPLACE - volume 手->股 (×100, 与 daily_update_xtdata 同口径) - 无限流, 单进程 download 不并发 -用法: python xt_eod.py [--limit N] [--dry-run] +用法: + python xt_eod.py [--limit N] [--dry-run] # 日常增量 (LOOKBACK=30 天) + python xt_eod.py --full-bj # 北交所 backfill (start=20240101) """ import argparse import datetime as dt import sqlite3 import time -from xtquant import xtdata as xd +try: + from xtquant import xtdata as xd +except ImportError: + xd = None # mac 单测 exc_of 时 xd=None, VPS 跑 main() 会 return 2 import pandas as pd from dbbardata_utils import normalize_daily_dt @@ -37,6 +42,9 @@ def prefix_of(sym): def exc_of(sym): + # 920 是 3 位前缀 (北交所), 必须在 2 位 SSE/SZSE 判断前优先, 否则 sym[:2]='92' 落 SZSE + if sym[:3] == "920": + return "BJSE" return "SSE" if sym[:2] in ("51", "56", "58", "50", "60", "68") else "SZSE" @@ -44,31 +52,60 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument("--limit", type=int, default=0) ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--full-bj", action="store_true", + help="北交所 920xxx backfill: start=20240101 (默认与 ETF 同 LOOKBACK)") args = ap.parse_args() - end = dt.datetime.now().strftime("%Y%m%d") - start = (dt.datetime.now() - dt.timedelta(days=LOOKBACK)).strftime("%Y%m%d") - log(f"start window={start}~{end}") + if xd is None: + log("FATAL xtquant 未装(VPS-only)") + return 2 - u = list(set( + end = dt.datetime.now().strftime("%Y%m%d") + etf_start = (dt.datetime.now() - dt.timedelta(days=LOOKBACK)).strftime("%Y%m%d") + bj_start = "20240101" if args.full_bj else etf_start + log(f"start window ETF/基金={etf_start} 北交所={bj_start}~{end} (full_bj={args.full_bj})") + + # 沪深 ETF/基金 (xtata 独占, baostock 不覆盖) + etf_codes = list(set( (xd.get_stock_list_in_sector("沪深ETF") or []) + (xd.get_stock_list_in_sector("沪深基金") or []) )) + # 北交所 920xxx (中证2000 成份股, baostock 不覆盖, xtata 独占) + bj_codes = [] + try: + _c = sqlite3.connect(DB, timeout=30) + bj_raw = [r[0] for r in _c.execute( + "SELECT DISTINCT code FROM constituent_unified " + "WHERE index_code='932000' AND code LIKE '920%'" + )] + _c.close() + bj_codes = [f"{c}.BJ" for c in bj_raw] + except Exception as e: + log(f"WARN constituent_unified 920 read err: {e}") + log(f"universe ETF/基金={len(etf_codes)} 北交所={len(bj_codes)}") + + u = etf_codes + bj_codes if not u: log("FATAL empty universe (miniQMT 未连?)") return 2 if args.limit: u = u[:args.limit] - log(f"universe ETF+基金: {len(u)}") - # download paced + def _start_of(code): + return bj_start if code.split(".")[0].startswith("920") else etf_start + + # download paced: 按 start 分组避免 download_history_data2 单 start 限制 BATCH = 200 - for i in range(0, len(u), BATCH): - try: - xd.download_history_data2(u[i:i+BATCH], "1d", start, end, lambda d, p: None) - except Exception as e: - log(f"dl batch@{i} err: {e}") - time.sleep(1.0) + for st in ({etf_start, bj_start}): + sub = [c for c in u if _start_of(c) == st] + if not sub: + continue + for i in range(0, len(sub), BATCH): + try: + xd.download_history_data2(sub[i:i+BATCH], "1d", st, end, lambda d, p: None) + except Exception as e: + log(f"dl @{st} @{i} err: {e}") + time.sleep(1.0) log("download done") conn = sqlite3.connect(DB, timeout=60) @@ -81,7 +118,8 @@ def main(): for i, code in enumerate(u): sym = code.split(".")[0] try: - r = xd.get_market_data_ex([], [code], period="1d", start_time=start, + r = xd.get_market_data_ex([], [code], period="1d", + start_time=_start_of(code), end_time=end, dividend_type="none") df = r.get(code) if r else None if df is None or not len(df): diff --git a/tests/data_platform/test_xt_eod_bj.py b/tests/data_platform/test_xt_eod_bj.py new file mode 100644 index 0000000..57e07d9 --- /dev/null +++ b/tests/data_platform/test_xt_eod_bj.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +"""TDD for xt_eod.exc_of — 北交所 920xxx exchange 判定扩展。 + +背景: + 中证2000(932000)有 39 只北交所成份股全 920xxx, baostock 完全不覆盖, + 由 xt_eod.py 用 xtdata 补全 (主力 ETF/基金管线之外的扩展)。 + 920 是 3 位前缀, 必须在原 2 位 SSE/SZSE 判断前优先识别, 否则 sym[:2]="92" + 会被错误归到 SZSE。 + +exc_of 是纯函数, 不依赖 xtquant, mac 无 xtquant 也可单测 (xt_eod.py 顶层 +xtquant import 失败时降级 xd=None, 仅 main() 受影响)。 +""" +import pytest + +from scripts.data_platform.xt_eod import exc_of + + +@pytest.mark.parametrize("sym,expected", [ + # 北交所 920xxx (3 位前缀优先, 任务核心) + ("920001", "BJSE"), + ("920002", "BJSE"), + ("920999", "BJSE"), + # 沪市 ETF/基金 (51/56/58/50) + ("510300", "SSE"), + ("588000", "SSE"), + ("561000", "SSE"), + ("501000", "SSE"), + # 沪市个股 (60/68) + ("600519", "SSE"), + ("688981", "SSE"), + # 深市个股/基金 (00/15/16/30 等) + ("000001", "SZSE"), + ("002001", "SZSE"), + ("300750", "SZSE"), + ("159915", "SZSE"), +]) +def test_exc_of(sym, expected): + assert exc_of(sym) == expected + + +def test_exc_of_3digit_priority_critical(): + """关键防坑: 920 必须在 2 位判断前拦截, 否则 sym[:2]='92' 落 SZSE。 + 这是本任务的核心 invariant — 单独显式断言一次。""" + assert exc_of("920001") == "BJSE" + # 同时确认 92 开头但非 920 仍归 SZSE (未来 921/929 等若出现, 走默认分支) + # 注意: 现网北交所目前只有 920 段;83/87/43 段归 BJ 应 separately handled if needed