feat(data): 双源5yr全市场部署 + baostock 15min + 断点续传

- raw_redownload 加断点续传(exists/skip已存在, 扩范围重下覆盖)
- baostock_download: 15min双源(qfq+raw)下载器, NAS容器跑, 限速防封
- full_deploy_5yr.sh: 无人值守日线双源pipeline(qfq→raw→rsync→验证)
- verify_dual_source: 容器内双源部署验证(fetch_day/iter_bars/除权日)

实测: 日线双源5yr 29600文件×2(rsync NAS), 浦发除权日 raw-5.9%/qfq-0.7%,
容器内 fetch_day/iter_bars/全市场抽检5只全通过. 15min沪深300 baostock限流121只.
This commit is contained in:
2026-07-09 19:22:36 +08:00
parent d5d3eea236
commit a372544045
4 changed files with 288 additions and 2 deletions
+20 -2
View File
@@ -79,6 +79,17 @@ def save_one(code: str, df) -> int:
return n
def exists(code: str, start_year: int) -> bool:
"""symbol 在 start_year 是否已有 parquet(断点续传)。
同范围续跑 → skip;扩范围(start 更早)→ 新 start_year 不存在 → 重下全量覆盖。
"""
pref = prefix_for(code)
return os.path.exists(
os.path.join(RAW_DIR, str(start_year), f"{pref}{code}_daily.parquet")
)
def load_all_codes(stock_list: str) -> list[str]:
"""从 stock_basic_info csv 读代码列表(容错列名)。"""
codes = []
@@ -101,6 +112,7 @@ def main():
ap.add_argument("--start", default="2024-01-01")
ap.add_argument("--end", default=None, help="默认今天")
ap.add_argument("--adjust", default="", help="复权: '' raw / 'qfq' 前复权(双源用)")
ap.add_argument("--force", action="store_true", help="强制重下(默认 skip 已存在=断点续传)")
args = ap.parse_args()
end = args.end or time.strftime("%Y-%m-%d")
@@ -121,8 +133,14 @@ def main():
import warnings
warnings.filterwarnings("ignore")
ok = fail = rows = 0
start_year = int(args.start[:4])
ok = fail = rows = skipped = 0
for i, code in enumerate(codes, 1):
if not args.force and exists(code, start_year):
skipped += 1
if skipped % 500 == 0:
log.info("[%d/%d] ... skipped %d 已存在", i, len(codes), skipped)
continue
df, err = download_one(ak, code, args.start, end, args.adjust)
if df is None:
fail += 1
@@ -138,7 +156,7 @@ def main():
log.error("[%d/%d] %s SAVE FAIL %s", i, len(codes), code, e)
time.sleep(SLEEP) # 限速(用户约束)
log.info("=== 完成: ok=%d fail=%d rows=%draw_dir=%s ===", ok, fail, rows, RAW_DIR)
log.info("=== 完成: ok=%d skip=%d fail=%d rows=%draw_dir=%s ===", ok, skipped, fail, rows, RAW_DIR)
if __name__ == "__main__":