feat(data): 双源5yr全市场部署 + baostock 15min + 断点续传
- raw_redownload 加断点续传(exists/skip已存在, 扩范围重下覆盖) - baostock_download: 15min双源(qfq+raw)下载器, NAS容器跑, 限速防封 - full_deploy_5yr.sh: 无人值守日线双源pipeline(qfq→raw→rsync→验证) - verify_dual_source: 容器内双源部署验证(fetch_day/iter_bars/除权日) 实测: 日线双源5yr 29600文件×2(rsync NAS), 浦发除权日 raw-5.9%/qfq-0.7%, 容器内 fetch_day/iter_bars/全市场抽检5只全通过. 15min沪深300 baostock限流121只.
This commit is contained in:
@@ -79,6 +79,17 @@ def save_one(code: str, df) -> int:
|
||||
return n
|
||||
|
||||
|
||||
def exists(code: str, start_year: int) -> bool:
|
||||
"""symbol 在 start_year 是否已有 parquet(断点续传)。
|
||||
|
||||
同范围续跑 → skip;扩范围(start 更早)→ 新 start_year 不存在 → 重下全量覆盖。
|
||||
"""
|
||||
pref = prefix_for(code)
|
||||
return os.path.exists(
|
||||
os.path.join(RAW_DIR, str(start_year), f"{pref}{code}_daily.parquet")
|
||||
)
|
||||
|
||||
|
||||
def load_all_codes(stock_list: str) -> list[str]:
|
||||
"""从 stock_basic_info csv 读代码列表(容错列名)。"""
|
||||
codes = []
|
||||
@@ -101,6 +112,7 @@ def main():
|
||||
ap.add_argument("--start", default="2024-01-01")
|
||||
ap.add_argument("--end", default=None, help="默认今天")
|
||||
ap.add_argument("--adjust", default="", help="复权: '' raw / 'qfq' 前复权(双源用)")
|
||||
ap.add_argument("--force", action="store_true", help="强制重下(默认 skip 已存在=断点续传)")
|
||||
args = ap.parse_args()
|
||||
|
||||
end = args.end or time.strftime("%Y-%m-%d")
|
||||
@@ -121,8 +133,14 @@ def main():
|
||||
import warnings
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
ok = fail = rows = 0
|
||||
start_year = int(args.start[:4])
|
||||
ok = fail = rows = skipped = 0
|
||||
for i, code in enumerate(codes, 1):
|
||||
if not args.force and exists(code, start_year):
|
||||
skipped += 1
|
||||
if skipped % 500 == 0:
|
||||
log.info("[%d/%d] ... skipped %d 已存在", i, len(codes), skipped)
|
||||
continue
|
||||
df, err = download_one(ak, code, args.start, end, args.adjust)
|
||||
if df is None:
|
||||
fail += 1
|
||||
@@ -138,7 +156,7 @@ def main():
|
||||
log.error("[%d/%d] %s SAVE FAIL %s", i, len(codes), code, e)
|
||||
time.sleep(SLEEP) # 限速(用户约束)
|
||||
|
||||
log.info("=== 完成: ok=%d fail=%d rows=%d,raw_dir=%s ===", ok, fail, rows, RAW_DIR)
|
||||
log.info("=== 完成: ok=%d skip=%d fail=%d rows=%d,raw_dir=%s ===", ok, skipped, fail, rows, RAW_DIR)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user