feat(data): xt_eod 扩展补全北交所 920xxx 日线(exc_of+universe+import容错,复用×100 normalize)
- exc_of: 920 -> BJSE (3 位前缀优先于 2 位 SSE/SZSE 判断, 防 sym[:2]='92' 落 SZSE) - universe: 沪深ETF/基金 ∪ 北交所920xxx (从 constituent_unified 932000 取, baostock 不覆盖) - import xtquant 容错 (mac xd=None 可单测 exc_of, main() 开头 return 2) - --full-bj: 北交所 backfill start=20240101 (默认 LOOKBACK=30 与 ETF 同窗) - 复用 normalize_daily_dt + ×100 volume 口径, 不动已有 ETF 写入路径 Tests: tests/data_platform/test_xt_eod_bj.py 14 cases RED -> GREEN (3 位前缀 critical + ETF/沪深/深市 覆盖)
This commit is contained in:
@@ -1,24 +1,29 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""xt_eod.py — sanguo-xt-eod (方案A schtask 18:40): xtata ETF/基金 EOD 增量 -> dbbardata('d')。
|
||||
"""xt_eod.py — sanguo-xt-eod (方案A schtask 18:40): xtata ETF/基金/北交所 EOD 增量 -> dbbardata('d')。
|
||||
|
||||
baostock 只取 type=1 股票, 不覆盖 ETF/基金 -> xtata 独占 (spec §14)。
|
||||
个股日线由 bs_eod (baostock) 灌, 此处不碰个股, 避免 dbbardata 两源冲突。
|
||||
baostock 只取 type=1 股票, 不覆盖 ETF/基金/北交所个股 -> xtata 独占 (spec §14)。
|
||||
北交所个股 baostock 不覆盖, 此处补; 沪深个股仍由 bs_eod 灌, 避免 dbbardata 两源冲突。
|
||||
|
||||
- universe = 沪深ETF ∪ 沪深基金
|
||||
- universe = 沪深ETF ∪ 沪深基金 ∪ 北交所920xxx (中证2000 成份股, baostock 不覆盖, xtata 独占)
|
||||
- download_history_data2 批量 paced -> 本地缓存
|
||||
- get_market_data_ex raw(dividend_type=none) -> dbbardata('d') INSERT OR REPLACE
|
||||
- volume 手->股 (×100, 与 daily_update_xtdata 同口径)
|
||||
- 无限流, 单进程 download 不并发
|
||||
|
||||
用法: python xt_eod.py [--limit N] [--dry-run]
|
||||
用法:
|
||||
python xt_eod.py [--limit N] [--dry-run] # 日常增量 (LOOKBACK=30 天)
|
||||
python xt_eod.py --full-bj # 北交所 backfill (start=20240101)
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
try:
|
||||
from xtquant import xtdata as xd
|
||||
except ImportError:
|
||||
xd = None # mac 单测 exc_of 时 xd=None, VPS 跑 main() 会 return 2
|
||||
import pandas as pd
|
||||
|
||||
from dbbardata_utils import normalize_daily_dt
|
||||
@@ -37,6 +42,9 @@ def prefix_of(sym):
|
||||
|
||||
|
||||
def exc_of(sym):
|
||||
# 920 是 3 位前缀 (北交所), 必须在 2 位 SSE/SZSE 判断前优先, 否则 sym[:2]='92' 落 SZSE
|
||||
if sym[:3] == "920":
|
||||
return "BJSE"
|
||||
return "SSE" if sym[:2] in ("51", "56", "58", "50", "60", "68") else "SZSE"
|
||||
|
||||
|
||||
@@ -44,30 +52,59 @@ def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--limit", type=int, default=0)
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--full-bj", action="store_true",
|
||||
help="北交所 920xxx backfill: start=20240101 (默认与 ETF 同 LOOKBACK)")
|
||||
args = ap.parse_args()
|
||||
|
||||
end = dt.datetime.now().strftime("%Y%m%d")
|
||||
start = (dt.datetime.now() - dt.timedelta(days=LOOKBACK)).strftime("%Y%m%d")
|
||||
log(f"start window={start}~{end}")
|
||||
if xd is None:
|
||||
log("FATAL xtquant 未装(VPS-only)")
|
||||
return 2
|
||||
|
||||
u = list(set(
|
||||
end = dt.datetime.now().strftime("%Y%m%d")
|
||||
etf_start = (dt.datetime.now() - dt.timedelta(days=LOOKBACK)).strftime("%Y%m%d")
|
||||
bj_start = "20240101" if args.full_bj else etf_start
|
||||
log(f"start window ETF/基金={etf_start} 北交所={bj_start}~{end} (full_bj={args.full_bj})")
|
||||
|
||||
# 沪深 ETF/基金 (xtata 独占, baostock 不覆盖)
|
||||
etf_codes = list(set(
|
||||
(xd.get_stock_list_in_sector("沪深ETF") or []) +
|
||||
(xd.get_stock_list_in_sector("沪深基金") or [])
|
||||
))
|
||||
# 北交所 920xxx (中证2000 成份股, baostock 不覆盖, xtata 独占)
|
||||
bj_codes = []
|
||||
try:
|
||||
_c = sqlite3.connect(DB, timeout=30)
|
||||
bj_raw = [r[0] for r in _c.execute(
|
||||
"SELECT DISTINCT code FROM constituent_unified "
|
||||
"WHERE index_code='932000' AND code LIKE '920%'"
|
||||
)]
|
||||
_c.close()
|
||||
bj_codes = [f"{c}.BJ" for c in bj_raw]
|
||||
except Exception as e:
|
||||
log(f"WARN constituent_unified 920 read err: {e}")
|
||||
log(f"universe ETF/基金={len(etf_codes)} 北交所={len(bj_codes)}")
|
||||
|
||||
u = etf_codes + bj_codes
|
||||
if not u:
|
||||
log("FATAL empty universe (miniQMT 未连?)")
|
||||
return 2
|
||||
if args.limit:
|
||||
u = u[:args.limit]
|
||||
log(f"universe ETF+基金: {len(u)}")
|
||||
|
||||
# download paced
|
||||
def _start_of(code):
|
||||
return bj_start if code.split(".")[0].startswith("920") else etf_start
|
||||
|
||||
# download paced: 按 start 分组避免 download_history_data2 单 start 限制
|
||||
BATCH = 200
|
||||
for i in range(0, len(u), BATCH):
|
||||
for st in ({etf_start, bj_start}):
|
||||
sub = [c for c in u if _start_of(c) == st]
|
||||
if not sub:
|
||||
continue
|
||||
for i in range(0, len(sub), BATCH):
|
||||
try:
|
||||
xd.download_history_data2(u[i:i+BATCH], "1d", start, end, lambda d, p: None)
|
||||
xd.download_history_data2(sub[i:i+BATCH], "1d", st, end, lambda d, p: None)
|
||||
except Exception as e:
|
||||
log(f"dl batch@{i} err: {e}")
|
||||
log(f"dl @{st} @{i} err: {e}")
|
||||
time.sleep(1.0)
|
||||
log("download done")
|
||||
|
||||
@@ -81,7 +118,8 @@ def main():
|
||||
for i, code in enumerate(u):
|
||||
sym = code.split(".")[0]
|
||||
try:
|
||||
r = xd.get_market_data_ex([], [code], period="1d", start_time=start,
|
||||
r = xd.get_market_data_ex([], [code], period="1d",
|
||||
start_time=_start_of(code),
|
||||
end_time=end, dividend_type="none")
|
||||
df = r.get(code) if r else None
|
||||
if df is None or not len(df):
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""TDD for xt_eod.exc_of — 北交所 920xxx exchange 判定扩展。
|
||||
|
||||
背景:
|
||||
中证2000(932000)有 39 只北交所成份股全 920xxx, baostock 完全不覆盖,
|
||||
由 xt_eod.py 用 xtdata 补全 (主力 ETF/基金管线之外的扩展)。
|
||||
920 是 3 位前缀, 必须在原 2 位 SSE/SZSE 判断前优先识别, 否则 sym[:2]="92"
|
||||
会被错误归到 SZSE。
|
||||
|
||||
exc_of 是纯函数, 不依赖 xtquant, mac 无 xtquant 也可单测 (xt_eod.py 顶层
|
||||
xtquant import 失败时降级 xd=None, 仅 main() 受影响)。
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from scripts.data_platform.xt_eod import exc_of
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sym,expected", [
|
||||
# 北交所 920xxx (3 位前缀优先, 任务核心)
|
||||
("920001", "BJSE"),
|
||||
("920002", "BJSE"),
|
||||
("920999", "BJSE"),
|
||||
# 沪市 ETF/基金 (51/56/58/50)
|
||||
("510300", "SSE"),
|
||||
("588000", "SSE"),
|
||||
("561000", "SSE"),
|
||||
("501000", "SSE"),
|
||||
# 沪市个股 (60/68)
|
||||
("600519", "SSE"),
|
||||
("688981", "SSE"),
|
||||
# 深市个股/基金 (00/15/16/30 等)
|
||||
("000001", "SZSE"),
|
||||
("002001", "SZSE"),
|
||||
("300750", "SZSE"),
|
||||
("159915", "SZSE"),
|
||||
])
|
||||
def test_exc_of(sym, expected):
|
||||
assert exc_of(sym) == expected
|
||||
|
||||
|
||||
def test_exc_of_3digit_priority_critical():
|
||||
"""关键防坑: 920 必须在 2 位判断前拦截, 否则 sym[:2]='92' 落 SZSE。
|
||||
这是本任务的核心 invariant — 单独显式断言一次。"""
|
||||
assert exc_of("920001") == "BJSE"
|
||||
# 同时确认 92 开头但非 920 仍归 SZSE (未来 921/929 等若出现, 走默认分支)
|
||||
# 注意: 现网北交所目前只有 920 段;83/87/43 段归 BJ 应 separately handled if needed
|
||||
Reference in New Issue
Block a user