claude_dev
c2a89d01a4
spec §14 方案A 数据层迁移完成 + E2E 验证(read_db_daily: 在市/退市治偏差/ETF 全OK):
- dbbardata('d') 1826万含退市(治回测幸存者偏差, INSERT OR REPLACE staging迁移, WHERE OHLC NOT NULL+COALESCE)
- constituent_unified 7110行/9指数(300/500/50 baostock全集 + 深证4指 akshare cni union + 中证1000/2000 snapshot)
- pe/pb 不进DB -> valuation_baostock/<year>.parquet 按年宽表(2003-2026)
- 废弃 daily_baostock_full/bs_index_constituent(rename _old 保留); 旧4 schtask disabled
- 新 schtask sanguo-bs-eod 18:05(baostock个股日线+15min+拆pe/pb DAILY_LIMIT 48000) + sanguo-xt-eod 18:40(ETF/基金xtata)
- 权威源: baostock个股日线+估值+15min+复权+300/500/50 / xtata ETF+基金+当天实时 / akshare三表+事件+深证中证成份股
- 全程备份+staging+_old保留可回滚; 脚本 audit/probe/migrate/merge/cleanup/fix_config/verify/bs_eod/xt_eod/wrapper/register_schtasks
- 待办(spec §6 使用层): LocalParquetProvider 接 constituent_unified+valuation_baostock + 实时拼接
76 lines
3.0 KiB
Python
76 lines
3.0 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""merge_daily_baostock.py — 单元4 合并: staging->dbbardata + daily_baostock_full->_old + 补 parquet。
|
|
|
|
前置: migrate_daily_baostock.py 已灌 dbbardata_staging_daily (1826万, verify OK);
|
|
dbbardata 有 UNIQUE(symbol,exchange,interval,datetime) (probe 确认)。
|
|
1. INSERT OR REPLACE staging -> dbbardata('d'): 个股(含退市) REPLACE 在市/新增退市 (治偏差)
|
|
2. daily_baostock_full -> daily_baostock_full_old (保留, pe/pb 源)
|
|
3. pe/pb -> data/valuation_baostock/<year>.parquet (从 _old, isST->int 修 ArrowTypeError)
|
|
回滚: rename daily_baostock_full_old->daily_baostock_full; dbbardata 从 .bak 恢复
|
|
"""
|
|
import sqlite3
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
DB = Path(r"C:\sanguo_vnpy_v2\data\quant_trading.db")
|
|
VAL_DIR = Path(r"C:\sanguo_vnpy_v2\data\valuation_baostock")
|
|
|
|
|
|
def log(m):
|
|
print(m, flush=True)
|
|
|
|
|
|
VAL_DIR.mkdir(parents=True, exist_ok=True)
|
|
c = sqlite3.connect(str(DB), timeout=600)
|
|
c.execute("PRAGMA busy_timeout = 600000")
|
|
c.execute("PRAGMA synchronous = NORMAL")
|
|
|
|
log("1. INSERT OR REPLACE staging(1826万) -> dbbardata('d') 含退市治偏差 ...")
|
|
# dbbardata 全列 NOT NULL; 跳过停牌(OHLC NULL), volume/turnover COALESCE 0
|
|
c.execute(
|
|
"INSERT OR REPLACE INTO dbbardata "
|
|
"(symbol,exchange,datetime,interval,volume,turnover,open_interest,"
|
|
"open_price,high_price,low_price,close_price) "
|
|
"SELECT symbol, exchange, datetime, interval, "
|
|
"COALESCE(volume, 0), COALESCE(turnover, 0), 0, "
|
|
"open_price, high_price, low_price, close_price "
|
|
"FROM dbbardata_staging_daily "
|
|
"WHERE open_price IS NOT NULL AND high_price IS NOT NULL "
|
|
"AND low_price IS NOT NULL AND close_price IS NOT NULL")
|
|
c.commit()
|
|
log(" INSERT OR REPLACE done")
|
|
|
|
log("2. daily_baostock_full -> daily_baostock_full_old")
|
|
c.execute("ALTER TABLE daily_baostock_full RENAME TO daily_baostock_full_old")
|
|
c.commit()
|
|
|
|
log("3. pe/pb -> valuation_baostock/<year>.parquet (isST->int, 流式避 OOM)")
|
|
val_by_year = {}
|
|
n = 0
|
|
for chunk in pd.read_sql(
|
|
"SELECT symbol,exchange,date,peTTM,psTTM,pcfNcfTTM,pbMRQ,turn,pctChg,isST "
|
|
"FROM daily_baostock_full_old", c, chunksize=200000):
|
|
n += 1
|
|
chunk["isST"] = pd.to_numeric(chunk["isST"], errors="coerce").fillna(0).astype(int)
|
|
chunk["_y"] = pd.to_datetime(chunk["date"]).dt.year
|
|
for yr, sub in chunk.groupby("_y"):
|
|
sub = sub.drop(columns=["_y"])
|
|
val_by_year.setdefault(int(yr), []).append(sub)
|
|
if n % 10 == 0:
|
|
log(f" parquet chunk#{n}")
|
|
for yr in sorted(val_by_year):
|
|
df_y = pd.concat(val_by_year[yr], ignore_index=True).sort_values(["symbol", "date"])
|
|
df_y.to_parquet(VAL_DIR / f"{yr}.parquet", index=False)
|
|
log(f" {yr}: {len(df_y)} rows")
|
|
|
|
log("4. verify dbbardata('d'):")
|
|
for sym, label in [("000005", "退市"), ("000023", "退市"), ("600519", "在市"), ("510300", "ETF")]:
|
|
r = c.execute(
|
|
"SELECT COUNT(*), MAX(datetime) FROM dbbardata "
|
|
"WHERE symbol=? AND interval='d'", (sym,)).fetchone()
|
|
log(f" {sym}({label}): {r}")
|
|
c.close()
|
|
log("MERGE DONE")
|