feat(data): 全市场5m/15m下载+灌库+每日增量链
xtdata逐只下载(非批量download_history_data2,后者5201只触发xtquant死锁), 先5m后15m(15m依赖5m),看get bars判成败(ret=None正常非失败)。 download_15m_xtdata.py + relaunch_15m_wrapper.ps1(auto-restart兜底segfault)。 import_vnpy_minute_fast.py灌库(INSERT OR REPLACE,interval存5m/15m,8028万行)。 _run_daily.ps1加5m/15m增量段(每天16:30)。validate_import.py校验。 全市场5201只,5m 6020万/15m 2007万bar,2025-07-17~2026-07-17。
This commit is contained in:
@@ -0,0 +1,262 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""灌后校验：parquet 日线 ↔ quant_trading.db dbbardata 三项一致性检查。
|
||||
|
||||
独立可跑。读两个环境变量：
|
||||
VNPY_DB_PATH 默认 C:\\sanguo_vnpy_v2\\data\\quant_trading.db
|
||||
DAILY_DIR 默认 C:\\sanguo_vnpy_v2\\data\\raw
|
||||
|
||||
三项校验：
|
||||
A 最新日期相等: max(date) over raw parquet == max(datetime) where interval='d' from DB
|
||||
B 抽查 3 只股 OHLCV 逐值相等: 600519(SSE)/000001(SZSE)/000858(SZSE),
|
||||
各 parquet 最新 5 行 vs DB datetime 倒序 limit 5, date 对齐逐值比对 (abs<1e-4)
|
||||
C 当天数据合理性: DB 中 date==parquet_max 且 interval='d' 的所有行,
|
||||
无 close_price<=0、无 high_price<low_price、close_price 非 null
|
||||
|
||||
退出码: PASS=0 / FAIL=1。纯 stdlib + pandas (VPS 已装)。
|
||||
|
||||
Windows GBK 终端需 `python -X utf8 validate_import.py` 调用。
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
DB_PATH = os.environ.get('VNPY_DB_PATH', r'C:\sanguo_vnpy_v2\data\quant_trading.db')
|
||||
DAILY_DIR = os.environ.get('DAILY_DIR', r'C:\sanguo_vnpy_v2\data\raw')
|
||||
|
||||
# (symbol_code, exchange, filename_prefix) 三只蓝筹股做抽查
|
||||
SAMPLES = [
|
||||
('600519', 'SSE', 'sh'),
|
||||
('000001', 'SZSE', 'sz'),
|
||||
('000858', 'SZSE', 'sz'),
|
||||
]
|
||||
|
||||
|
||||
def norm_date_str(s):
|
||||
"""统一成 'YYYY-MM-DD' 字符串。接受 Timestamp / datetime / str。"""
|
||||
ts = pd.Timestamp(s)
|
||||
return ts.strftime('%Y-%m-%d')
|
||||
|
||||
|
||||
def find_parquet_max_date():
|
||||
"""扫最近年份目录下所有 raw parquet，取最大 date。
|
||||
只扫最新有数据的年份目录（~5000 文件 × 仅读 date 列 ≈ 10s）。
|
||||
返回 (max_date_str or None, n_files_scanned)。"""
|
||||
max_date = None
|
||||
n_files = 0
|
||||
year_dirs = sorted(
|
||||
[d for d in Path(DAILY_DIR).glob('*') if d.is_dir() and d.name.isdigit()],
|
||||
key=lambda d: d.name,
|
||||
reverse=True,
|
||||
)
|
||||
for year_dir in year_dirs[:2]: # 最多看最近 2 个年份目录
|
||||
local_max = None
|
||||
for f in year_dir.glob('*.parquet'):
|
||||
n_files += 1
|
||||
try:
|
||||
df = pd.read_parquet(f, columns=['date'])
|
||||
if df.empty:
|
||||
continue
|
||||
cur = df['date'].max()
|
||||
if local_max is None or cur > local_max:
|
||||
local_max = cur
|
||||
except Exception:
|
||||
continue
|
||||
if local_max is not None:
|
||||
max_date = local_max
|
||||
break # 当前年份有数据就不再看上一年
|
||||
return (norm_date_str(max_date) if max_date is not None else None, n_files)
|
||||
|
||||
|
||||
def check_a(parquet_max, db_max):
|
||||
"""A: 最新日期相等"""
|
||||
p = parquet_max
|
||||
d = db_max[:10] if db_max else None
|
||||
ok = (p is not None) and (d is not None) and (p == d)
|
||||
return ok, {'parquet_max': p, 'db_max': d}
|
||||
|
||||
|
||||
def check_b(conn):
|
||||
"""B: 3 只股 OHLCV 逐值比对"""
|
||||
detail = {}
|
||||
all_ok = True
|
||||
for code, exchange, prefix in SAMPLES:
|
||||
# 从最新年份目录往前找 parquet
|
||||
year_dirs = sorted(
|
||||
[d for d in Path(DAILY_DIR).glob('*') if d.is_dir() and d.name.isdigit()],
|
||||
key=lambda d: d.name,
|
||||
reverse=True,
|
||||
)
|
||||
parquet_path = None
|
||||
for yd in year_dirs:
|
||||
p = yd / f'{prefix}{code}_daily.parquet'
|
||||
if p.exists():
|
||||
parquet_path = p
|
||||
break
|
||||
if parquet_path is None:
|
||||
detail[code] = {'status': 'MISSING_PARQUET'}
|
||||
all_ok = False
|
||||
continue
|
||||
try:
|
||||
pdf = pd.read_parquet(parquet_path).sort_values('date').tail(5).reset_index(drop=True)
|
||||
except Exception as e:
|
||||
detail[code] = {'status': f'PARQUET_READ_ERR: {e}'}
|
||||
all_ok = False
|
||||
continue
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT datetime, open_price, high_price, low_price, close_price, volume "
|
||||
"FROM dbbardata WHERE symbol=? AND exchange=? AND interval='d' "
|
||||
"ORDER BY datetime DESC LIMIT 5",
|
||||
(code, exchange),
|
||||
).fetchall()
|
||||
except Exception as e:
|
||||
detail[code] = {'status': f'DB_QUERY_ERR: {e}'}
|
||||
all_ok = False
|
||||
continue
|
||||
if not rows:
|
||||
detail[code] = {'status': 'DB_EMPTY'}
|
||||
all_ok = False
|
||||
continue
|
||||
db_pdf = pd.DataFrame(rows, columns=['datetime', 'open', 'high', 'low', 'close', 'volume'])
|
||||
db_pdf = db_pdf.sort_values('datetime').reset_index(drop=True)
|
||||
db_pdf['date'] = db_pdf['datetime'].str[:10]
|
||||
pdf = pdf.copy()
|
||||
pdf['date'] = pdf['date'].astype(str).str[:10]
|
||||
|
||||
# 取共同尾部 n 行
|
||||
n = min(len(pdf), len(db_pdf))
|
||||
if n == 0:
|
||||
detail[code] = {'status': 'EMPTY_AFTER_ALIGN'}
|
||||
all_ok = False
|
||||
continue
|
||||
p_tail = pdf.tail(n).reset_index(drop=True)
|
||||
d_tail = db_pdf.tail(n).reset_index(drop=True)
|
||||
|
||||
dates_match = (p_tail['date'].values == d_tail['date'].values).all()
|
||||
cols = ['open', 'high', 'low', 'close', 'volume']
|
||||
vals_match = True
|
||||
bad_col = None
|
||||
for c in cols:
|
||||
try:
|
||||
pv = p_tail[c].astype(float).values
|
||||
dv = d_tail[c].astype(float).values
|
||||
max_diff = float((abs(pv - dv)).max())
|
||||
if max_diff > 1e-4:
|
||||
vals_match = False
|
||||
bad_col = f'{c}(max_diff={max_diff})'
|
||||
break
|
||||
except Exception as e:
|
||||
vals_match = False
|
||||
bad_col = f'{c}: {e}'
|
||||
break
|
||||
|
||||
ok = bool(dates_match and vals_match)
|
||||
if not ok:
|
||||
all_ok = False
|
||||
detail[code] = {
|
||||
'status': 'PASS' if ok else 'FAIL',
|
||||
'n_rows': n,
|
||||
'dates_match': bool(dates_match),
|
||||
'vals_match': bool(vals_match),
|
||||
'bad_col': bad_col,
|
||||
'latest_date': p_tail['date'].iloc[-1] if len(p_tail) else None,
|
||||
'latest_close': float(p_tail['close'].iloc[-1]) if len(p_tail) else None,
|
||||
}
|
||||
return all_ok, detail
|
||||
|
||||
|
||||
def check_c(conn, parquet_max_str):
|
||||
"""C: 当天数据合理性（DB 中 date==parquet_max 的所有 interval='d' 行）"""
|
||||
if not parquet_max_str:
|
||||
return False, {'reason': 'no parquet_max'}
|
||||
pattern = parquet_max_str + '%'
|
||||
try:
|
||||
total = conn.execute(
|
||||
"SELECT COUNT(*) FROM dbbardata WHERE interval='d' AND datetime LIKE ?",
|
||||
(pattern,),
|
||||
).fetchone()[0]
|
||||
except Exception as e:
|
||||
return False, {'reason': f'DB_QUERY_ERR: {e}'}
|
||||
if total == 0:
|
||||
return False, {'reason': f'no rows for {parquet_max_str}'}
|
||||
try:
|
||||
bad_close = conn.execute(
|
||||
"SELECT COUNT(*) FROM dbbardata WHERE interval='d' AND datetime LIKE ? "
|
||||
"AND (close_price IS NULL OR close_price <= 0)",
|
||||
(pattern,),
|
||||
).fetchone()[0]
|
||||
bad_hl = conn.execute(
|
||||
"SELECT COUNT(*) FROM dbbardata WHERE interval='d' AND datetime LIKE ? "
|
||||
"AND high_price < low_price",
|
||||
(pattern,),
|
||||
).fetchone()[0]
|
||||
except Exception as e:
|
||||
return False, {'reason': f'DB_QUERY_ERR: {e}'}
|
||||
ok = (bad_close == 0) and (bad_hl == 0)
|
||||
return ok, {
|
||||
'date': parquet_max_str,
|
||||
'total_rows': total,
|
||||
'bad_close': bad_close,
|
||||
'bad_high_low': bad_hl,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
print(f'[VALIDATE] DB={DB_PATH}')
|
||||
print(f'[VALIDATE] DAILY_DIR={DAILY_DIR}')
|
||||
|
||||
if not os.path.exists(DB_PATH):
|
||||
print(f'[VALIDATE][FAIL] DB not found: {DB_PATH}')
|
||||
sys.exit(1)
|
||||
if not os.path.exists(DAILY_DIR):
|
||||
print(f'[VALIDATE][FAIL] DAILY_DIR not found: {DAILY_DIR}')
|
||||
sys.exit(1)
|
||||
|
||||
parquet_max, n_files = find_parquet_max_date()
|
||||
if parquet_max is None:
|
||||
print(f'[VALIDATE][FAIL] no parquet under {DAILY_DIR} (scanned {n_files} files)')
|
||||
sys.exit(1)
|
||||
print(f'[VALIDATE] scanned {n_files} parquet files, parquet_max={parquet_max}')
|
||||
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
|
||||
try:
|
||||
row = conn.execute("SELECT MAX(datetime) FROM dbbardata WHERE interval='d'").fetchone()
|
||||
db_max = row[0] if row else None
|
||||
except Exception as e:
|
||||
print(f'[VALIDATE][FAIL] DB query max(datetime) err: {e}')
|
||||
conn.close()
|
||||
sys.exit(1)
|
||||
print(f'[VALIDATE] db_max={db_max}')
|
||||
|
||||
# A
|
||||
a_ok, a_detail = check_a(parquet_max, db_max)
|
||||
print(f'[VALIDATE][A] {"PASS" if a_ok else "FAIL"} parquet_max={a_detail["parquet_max"]} db_max={a_detail["db_max"]}')
|
||||
|
||||
# B
|
||||
b_ok, b_detail = check_b(conn)
|
||||
print(f'[VALIDATE][B] {"PASS" if b_ok else "FAIL"}')
|
||||
for code, info in b_detail.items():
|
||||
if info.get('status') == 'PASS':
|
||||
print(f' {code}: PASS n={info["n_rows"]} latest={info["latest_date"]} close={info["latest_close"]}')
|
||||
else:
|
||||
print(f' {code}: {info}')
|
||||
|
||||
# C
|
||||
c_ok, c_detail = check_c(conn, parquet_max)
|
||||
print(f'[VALIDATE][C] {"PASS" if c_ok else "FAIL"} {c_detail}')
|
||||
|
||||
conn.close()
|
||||
|
||||
overall = a_ok and b_ok and c_ok
|
||||
print(f'[VALIDATE] {"PASS" if overall else "FAIL"} '
|
||||
f'(A={"PASS" if a_ok else "FAIL"} B={"PASS" if b_ok else "FAIL"} C={"PASS" if c_ok else "FAIL"})')
|
||||
sys.exit(0 if overall else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user