claude_dev
cf4b3a2cb0
- download_minute.py: STOCK_ROOT环境变量参数化(Mac/NAS兼容); 修tencent备源amount聚合 last→sum - refresh_15min_daily.py: 新建cron入口(交易日判断baostock/直连pop代理/日志) - 回填470缺口(170 SH/SZ成功, 300 BSE源不支持) - docs/data/15min-data-design.md: 15min数据层设计文档(源/覆盖/脚本/刷新/约束/路径映射)
131 lines
4.8 KiB
Python
Executable File
131 lines
4.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""
|
||
15min 日增量刷新 - NAS Synology 任务计划入口
|
||
|
||
功能：
|
||
1. 清除代理环境变量（直连，避免被识别为异常流量）
|
||
2. 判断今天是否 A 股交易日（baostock query_trade_dates；不可用降级为"周六日跳过"）
|
||
3. 交易日 → 调 download_minute.py --scope all --resume
|
||
4. 记日志到 $STOCK_ROOT/logs/daily_update/refresh_15min_YYYYMMDD.log
|
||
|
||
环境变量：
|
||
STOCK_ROOT 数据根目录（默认 /volume1/stock，Mac 测试用 /Volumes/stock）
|
||
|
||
Synology 任务计划命令（容器内 Python，依赖已装好）：
|
||
docker exec sanguo_vnpy_v2 bash -c "cd /app && python3 scripts/data_platform/refresh_15min_daily.py"
|
||
|
||
或 NAS host Python（需自行 pip install baostock pandas pyarrow）：
|
||
STOCK_ROOT=/volume1/stock python3 /volume1/homes/admin/.sanguo_projects/sanguo_vnpy_v2/scripts/data_platform/refresh_15min_daily.py
|
||
"""
|
||
|
||
import logging
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Tuple
|
||
|
||
# --- 清除代理（脚本一进来就 pop，保证后续 urllib/baostock 直连）---
|
||
for _k in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY",
|
||
"all_proxy", "ALL_PROXY", "no_proxy", "NO_PROXY"):
|
||
os.environ.pop(_k, None)
|
||
|
||
# --- 路径配置 ---
|
||
# NAS: /volume1/stock；Mac: /Volumes/stock。也可通过环境变量覆盖。
|
||
STOCK_ROOT = Path(os.environ.get("STOCK_ROOT", "/volume1/stock"))
|
||
LOG_DIR = STOCK_ROOT / "logs" / "daily_update"
|
||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||
DOWNLOAD_SCRIPT = SCRIPT_DIR / "download_minute.py"
|
||
|
||
# --- 日志（先建目录，LOG_DIR 在 STOCK_ROOT 下） ---
|
||
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||
_today_str = datetime.now().strftime("%Y%m%d")
|
||
LOG_FILE = LOG_DIR / f"refresh_15min_{_today_str}.log"
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s %(levelname)s %(message)s",
|
||
handlers=[
|
||
logging.FileHandler(LOG_FILE, encoding="utf-8"),
|
||
logging.StreamHandler(),
|
||
],
|
||
)
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
def is_trading_day() -> Tuple[bool, str]:
|
||
"""
|
||
判断今天是否 A 股交易日。
|
||
优先 baostock query_trade_dates；不可用降级为"周六日跳过"（工作日默认交易）。
|
||
返回 (is_trading, reason)
|
||
"""
|
||
today_iso = datetime.now().strftime("%Y-%m-%d")
|
||
today_date = datetime.now().date()
|
||
|
||
# 周末一定非交易日（即使 baostock 失败也能正确跳过）
|
||
if today_date.weekday() >= 5:
|
||
return False, f"周末({today_date.strftime('%A')})"
|
||
|
||
# 工作日 → 用 baostock 查节假日
|
||
try:
|
||
import baostock as bs
|
||
lg = bs.login()
|
||
if lg.error_code != "0":
|
||
logger.warning("baostock 登录失败: %s，降级为工作日默认交易日", lg.error_msg)
|
||
return True, f"baostock登录失败({lg.error_msg})，工作日默认交易日"
|
||
try:
|
||
rs = bs.query_trade_dates(start_date=today_iso, end_date=today_iso)
|
||
if rs.error_code != "0":
|
||
logger.warning("baostock 查询交易日失败: %s，降级", rs.error_msg)
|
||
return True, f"baostock查询失败({rs.error_msg})，工作日默认交易日"
|
||
while rs.next():
|
||
row = rs.get_row_data() # [date, is_trading_day]
|
||
if row[0] == today_iso:
|
||
is_trading = (row[1] == "1")
|
||
return is_trading, f"baostock is_trading_day={row[1]}"
|
||
# 没查到今天 → 工作日默认交易
|
||
return True, "baostock 无今日记录，工作日默认交易日"
|
||
finally:
|
||
bs.logout()
|
||
except Exception as e:
|
||
logger.warning("baostock 不可用: %s，降级为工作日默认交易日", e)
|
||
return True, f"baostock异常({e})，工作日默认交易日"
|
||
|
||
|
||
def run_download() -> int:
|
||
"""调用 download_minute.py --scope all --resume，继承当前环境（含 STOCK_ROOT）"""
|
||
cmd = [sys.executable, str(DOWNLOAD_SCRIPT), "--scope", "all", "--resume"]
|
||
env = dict(os.environ)
|
||
env["STOCK_ROOT"] = str(STOCK_ROOT)
|
||
logger.info("执行命令: %s", " ".join(cmd))
|
||
logger.info("STOCK_ROOT=%s", STOCK_ROOT)
|
||
result = subprocess.run(cmd, env=env)
|
||
return result.returncode
|
||
|
||
|
||
def main() -> int:
|
||
logger.info("=" * 60)
|
||
logger.info("15min 日刷新开始 @ %s", datetime.now().isoformat())
|
||
logger.info("STOCK_ROOT=%s", STOCK_ROOT)
|
||
logger.info("LOG_FILE=%s", LOG_FILE)
|
||
|
||
trading, reason = is_trading_day()
|
||
logger.info("交易日判断: %s (%s)", trading, reason)
|
||
if not trading:
|
||
logger.info("非交易日, 跳过下载")
|
||
return 0
|
||
|
||
rc = run_download()
|
||
logger.info("download_minute.py 退出码: %d", rc)
|
||
if rc != 0:
|
||
logger.error("❌ 下载脚本失败")
|
||
return rc
|
||
|
||
logger.info("✅ 15min 日刷新完成")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|