58 lines
2.1 KiB
Python
58 lines
2.1 KiB
Python
# sanguo_data/scheduler.py
|
|
import json
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from sanguo_data.config import DataConfig
|
|
from sanguo_data.datafeed import fetch_daily
|
|
from sanguo_data.validator import validate_daily
|
|
from sanguo_data.datawriter import write_daily
|
|
|
|
@dataclass
|
|
class UpdateReport:
|
|
total: int = 0
|
|
success: int = 0
|
|
failed: int = 0
|
|
skipped: int = 0
|
|
failures: list = field(default_factory=list)
|
|
|
|
def run_daily_update(cfg: DataConfig, symbols: list[str] | None = None) -> UpdateReport:
|
|
progress_path = Path(cfg.data_paths.get("progress_file", "progress.json"))
|
|
progress = json.loads(progress_path.read_text()) if progress_path.exists() else {}
|
|
symbols = symbols or _load_stock_list(cfg)
|
|
report = UpdateReport(total=len(symbols))
|
|
fail_window = cfg.performance.get("fail_window", 100)
|
|
fail_threshold = cfg.performance.get("fail_threshold", 0.8)
|
|
|
|
for sym in symbols:
|
|
if progress.get(sym) == "done":
|
|
report.skipped += 1
|
|
continue
|
|
try:
|
|
df = fetch_daily(sym, _last_date(sym, cfg), _today(), cfg)
|
|
df = validate_daily(df)
|
|
if len(df) > 0:
|
|
write_daily(sym, df, cfg)
|
|
progress[sym] = "done"
|
|
progress_path.write_text(json.dumps(progress, ensure_ascii=False))
|
|
report.success += 1
|
|
except Exception as e:
|
|
report.failed += 1
|
|
report.failures.append({"symbol": sym, "error": str(e)})
|
|
checked = report.success + report.failed
|
|
if checked >= fail_window and report.failed / max(checked, 1) > fail_threshold:
|
|
report.failures.append({"error": "FAIL_THRESHOLD_REACHED, abort"})
|
|
break
|
|
time.sleep(cfg.performance.get("request_interval", 0.3))
|
|
return report
|
|
|
|
def _load_stock_list(cfg: DataConfig) -> list[str]:
|
|
"""从 v1 data_platform/daily_all_update.py copy 全市场股票列表读取"""
|
|
raise NotImplementedError("copy from v1")
|
|
|
|
def _last_date(symbol: str, cfg: DataConfig) -> str:
|
|
return "2020-01-01" # 简化，实际读 parquet 最后日期
|
|
|
|
def _today() -> str:
|
|
return "2026-07-05"
|