#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""merge_daily_baostock.py — 单元4 合并: staging->dbbardata + daily_baostock_full->_old + 补 parquet。

前置: migrate_daily_baostock.py 已灌 dbbardata_staging_daily (1826万, verify OK);
      dbbardata 有 UNIQUE(symbol,exchange,interval,datetime) (probe 确认)。
1. INSERT OR REPLACE staging -> dbbardata('d'): 个股(含退市) REPLACE 在市/新增退市 (治偏差)
2. daily_baostock_full -> daily_baostock_full_old (保留, pe/pb 源)
3. pe/pb -> data/valuation_baostock/<year>.parquet (从 _old, isST->int 修 ArrowTypeError)
回滚: rename daily_baostock_full_old->daily_baostock_full; dbbardata 从 .bak 恢复
"""
import sqlite3
from pathlib import Path

import pandas as pd

DB = Path(r"C:\sanguo_vnpy_v2\data\quant_trading.db")
VAL_DIR = Path(r"C:\sanguo_vnpy_v2\data\valuation_baostock")


def log(m):
    print(m, flush=True)


VAL_DIR.mkdir(parents=True, exist_ok=True)
c = sqlite3.connect(str(DB), timeout=600)
c.execute("PRAGMA busy_timeout = 600000")
c.execute("PRAGMA synchronous = NORMAL")

log("1. INSERT OR REPLACE staging(1826万) -> dbbardata('d') 含退市治偏差 ...")
# dbbardata 全列 NOT NULL; 跳过停牌(OHLC NULL), volume/turnover COALESCE 0
c.execute(
    "INSERT OR REPLACE INTO dbbardata "
    "(symbol,exchange,datetime,interval,volume,turnover,open_interest,"
    "open_price,high_price,low_price,close_price) "
    "SELECT symbol, exchange, datetime, interval, "
    "COALESCE(volume, 0), COALESCE(turnover, 0), 0, "
    "open_price, high_price, low_price, close_price "
    "FROM dbbardata_staging_daily "
    "WHERE open_price IS NOT NULL AND high_price IS NOT NULL "
    "AND low_price IS NOT NULL AND close_price IS NOT NULL")
c.commit()
log("   INSERT OR REPLACE done")

log("2. daily_baostock_full -> daily_baostock_full_old")
c.execute("ALTER TABLE daily_baostock_full RENAME TO daily_baostock_full_old")
c.commit()

log("3. pe/pb -> valuation_baostock/<year>.parquet (isST->int, 流式避 OOM)")
val_by_year = {}
n = 0
for chunk in pd.read_sql(
    "SELECT symbol,exchange,date,peTTM,psTTM,pcfNcfTTM,pbMRQ,turn,pctChg,isST "
    "FROM daily_baostock_full_old", c, chunksize=200000):
    n += 1
    chunk["isST"] = pd.to_numeric(chunk["isST"], errors="coerce").fillna(0).astype(int)
    chunk["_y"] = pd.to_datetime(chunk["date"]).dt.year
    for yr, sub in chunk.groupby("_y"):
        sub = sub.drop(columns=["_y"])
        val_by_year.setdefault(int(yr), []).append(sub)
    if n % 10 == 0:
        log(f"   parquet chunk#{n}")
for yr in sorted(val_by_year):
    df_y = pd.concat(val_by_year[yr], ignore_index=True).sort_values(["symbol", "date"])
    df_y.to_parquet(VAL_DIR / f"{yr}.parquet", index=False)
    log(f"   {yr}: {len(df_y)} rows")

log("4. verify dbbardata('d'):")
for sym, label in [("000005", "退市"), ("000023", "退市"), ("600519", "在市"), ("510300", "ETF")]:
    r = c.execute(
        "SELECT COUNT(*), MAX(datetime) FROM dbbardata "
        "WHERE symbol=? AND interval='d'", (sym,)).fetchone()
    log(f"   {sym}({label}): {r}")
c.close()
log("MERGE DONE")