feat(data): P0指数日K双源——bs_eod主循环后追加baostock指数段(2006+全量REPLACE)——治指数点位单源停更史(000300曾停7-16/000938停2023/932000三源无):①INDEX_CODES 15只=探针实证有数(300/500/1000/上证50/中证100/上证综指/深成/创业板+399303国证2000替932000+中证行业6只928/931-935;932000/000938/000985/929/930/936/937 baostock无→不进列表免每日warning刷屏)②同PK(symbol,exchange,datetime,interval)唯一索引INSERT OR REPLACE,与sina idx-eod存量行互备后写者胜,datetime归一纯日期同格式③段在主循环后:query达限跳过守DAILY_LIMIT预算;段内单指数fetch/upsert异常只log不外抛,退出码语义不变(schtask结果码可见)④fetch复用fetch_k_with_timeout(计QUERY_COUNT约15query/天可忽略)+relogin兜底;+10测试(sh/sz映射/REPLACE不双行/空行跳过/容错续跑/代码表卫生/达限跳过/正常跑调用);relogin计数3测试补run_index_eod隔离mock(指数段自身relogin是预期新行为);data_platform 126绿 [vps]
This commit is contained in:
@@ -5,6 +5,7 @@
|
||||
每日收盘后跑(VPS, baostock 日终更新就绪):
|
||||
- 个股日线(含退市, LOOKBACK 7 天) -> dbbardata('d') INSERT OR REPLACE (治幸存者偏差)
|
||||
- 个股 15min(LOOKBACK 7) -> dbbardata('15m')
|
||||
- 指数日K(2006+ 全量 REPLACE) -> dbbardata('d') 双源冗余, 主循环后跑不阻塞个股
|
||||
- 日线 pe/pb/turn/pctChg/isST -> data/valuation_baostock/<year>.parquet 追加
|
||||
- DAILY_LIMIT=48000 单进程单登录, sleep 0.3s, login 探针 graceful skip
|
||||
|
||||
@@ -50,6 +51,29 @@ DAILY_FIELDS = ("date,code,open,high,low,close,volume,amount,turn,"
|
||||
"pctChg,peTTM,psTTM,pcfNcfTTM,pbMRQ,isST")
|
||||
M15_FIELDS = "date,time,code,open,high,low,close,volume,amount"
|
||||
|
||||
# ======================== 指数日K双源 (2026-08-20 P0) ========================
|
||||
# baostock 指数 2006+ 与 sina idx-eod 互为冗余: 同 PK(symbol,exchange,datetime,
|
||||
# interval) INSERT OR REPLACE, 后写者胜 —— 治 000300 单源停更史(7-16)/932000 无点位
|
||||
# (baostock 也无 → 基准用 399303 国证2000 替代)/000938 停 2023(两源皆弃)。
|
||||
# 探针实证(2026-08-20 VPS): 下表代码全部有数; 932000/000938/000985/929/930/936/937
|
||||
# baostock 无 → 不进列表(免每日 warning 刷屏)。000016/399001/399006 未探针但属
|
||||
# 规模/成指类大概率有, rows=0 自动跳过不报错。
|
||||
INDEX_START = "2006-01-01"
|
||||
INDEX_FIELDS = "date,code,open,high,low,close,volume,amount"
|
||||
INDEX_CODES = (
|
||||
"sh.000001", # 上证综指
|
||||
"sh.000016", # 上证50
|
||||
"sh.000300", # 沪深300
|
||||
"sh.000905", # 中证500
|
||||
"sh.000852", # 中证1000
|
||||
"sh.000903", # 中证100
|
||||
"sz.399001", # 深证成指
|
||||
"sz.399006", # 创业板指
|
||||
"sz.399303", # 国证2000 (932000 中证2000 baostock 无, 以此作小盘基准)
|
||||
# 中证一级行业(baostock 可用的 6 只: 928/931/932/933/934/935)
|
||||
"sh.000928", "sh.000931", "sh.000932", "sh.000933", "sh.000934", "sh.000935",
|
||||
)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s",
|
||||
handlers=[logging.StreamHandler(sys.stdout)])
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -267,6 +291,62 @@ def upsert_15m(conn, code, prefix, rows):
|
||||
return len(db)
|
||||
|
||||
|
||||
def upsert_index_daily(conn, prefix, code, rows):
|
||||
"""指数日K rows -> dbbardata('d')。与 sina idx-eod 存量行同 PK, REPLACE 后写者胜。"""
|
||||
if not rows:
|
||||
return 0
|
||||
df = pd.DataFrame(rows, columns=INDEX_FIELDS.split(","))
|
||||
for c in ["open", "high", "low", "close", "volume", "amount"]:
|
||||
df[c] = pd.to_numeric(df[c], errors="coerce")
|
||||
exc = EXC_MAP[prefix]
|
||||
db = pd.DataFrame({
|
||||
"symbol": code, "exchange": exc,
|
||||
"datetime": df["date"].astype(str).map(normalize_daily_dt),
|
||||
"interval": "d", "volume": df["volume"], "turnover": df["amount"],
|
||||
"open_interest": 0.0,
|
||||
"open_price": df["open"], "high_price": df["high"],
|
||||
"low_price": df["low"], "close_price": df["close"],
|
||||
})
|
||||
conn.executemany(
|
||||
"INSERT OR REPLACE INTO dbbardata "
|
||||
"(symbol,exchange,datetime,interval,volume,turnover,open_interest,"
|
||||
"open_price,high_price,low_price,close_price) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
|
||||
db.itertuples(index=False, name=None))
|
||||
return len(db)
|
||||
|
||||
|
||||
def run_index_eod(conn, end):
|
||||
"""主循环后拉指数日K(辅助数据层): 单指数 fetch/upsert 失败只 log 不外抛。
|
||||
|
||||
指数是辅助数据 —— 任何异常不得影响个股 EOD 的退出码(schtask 结果码语义保持)。
|
||||
fetch_k_with_timeout 内部已计 QUERY_COUNT(指数 ~15 query/天, 预算可忽略)。
|
||||
"""
|
||||
n_ok = n_empty = 0
|
||||
for bs_code in INDEX_CODES:
|
||||
prefix, code = bs_code.split(".", 1)
|
||||
try:
|
||||
rows = fetch_k_with_timeout(bs_code, INDEX_FIELDS, "d", INDEX_START, end)
|
||||
except Exception as e:
|
||||
log.warning("指数 %s fetch err: %s", bs_code, e)
|
||||
if not relogin():
|
||||
log.error("指数段 relogin 失败, 提前结束本段")
|
||||
return
|
||||
continue
|
||||
try:
|
||||
with conn:
|
||||
n = upsert_index_daily(conn, prefix, code, rows)
|
||||
except Exception as e:
|
||||
log.warning("指数 %s upsert err: %s", bs_code, e)
|
||||
continue
|
||||
if n:
|
||||
n_ok += 1
|
||||
else:
|
||||
n_empty += 1
|
||||
log.warning("指数 %s 返回 0 行(源缺该指数?), 跳过", bs_code)
|
||||
log.info("[INDEX] ok=%d empty=%d (baostock 与 sina idx-eod 双源互备)",
|
||||
n_ok, n_empty)
|
||||
|
||||
|
||||
def _process_one_stock(conn, code, prefix, args, start, end):
|
||||
"""单只股票: fetch_k + upsert, 在 with conn 短事务里执行 (大事务根治).
|
||||
|
||||
@@ -357,6 +437,14 @@ def main():
|
||||
log.warning("周期 relogin 失败, 继续跑 (下次 fetch 失败时被动 relogin 兜底)")
|
||||
if i < len(stocks) - 1:
|
||||
time.sleep(BS_INTERVAL)
|
||||
# 指数日K双源段(主循环后): 达限跳过守预算; 段内异常全吞, 不改退出码
|
||||
if not limit_reached:
|
||||
try:
|
||||
run_index_eod(conn, end)
|
||||
except Exception as e:
|
||||
log.error("[INDEX] 段级异常(不影响个股 EOD 结果): %s", e)
|
||||
else:
|
||||
log.warning("query 达限, 跳过指数段(次日 2006+ 全量 REPLACE 自愈)")
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user