fix(data): corpus 全 fetcher 补 UA+jsonp 解析防御 [nas]
CI/CD / test (push) Successful in 20s
CI/CD / nas-deploy (push) Successful in 5s
CI/CD / nas-verify (push) Successful in 7s

NAS 冒烟二跑实锤: search-api 对无 UA 请求返 200+空 body(金丝雀当时带 UA 故
未暴露)→s.index('(') ValueError 裸崩。修: 统一 UA 常量六处 fetcher 全带;
jsonp 非 cb(...) 形态 → TransportError(带响应片段,计入域名连续错走冷却
而非裸崩);容器内 A/B 实测 no-UA='' vs withUA=正常610条
This commit is contained in:
2026-09-06 23:44:48 +08:00
parent 284ac472b1
commit ae6e2d92d0
+9 -6
View File
@@ -78,6 +78,7 @@ PDF_CAPACITY_CHECK_GB = 50 # 每落 50GB 自查一次 df
RATE = {"cninfo": 1.0, "eastmoney": 1.0, "article": 2.0} # 每域最小间隔(秒), 慢爬纪律
JITTER = (0.0, 0.3)
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
ID_KEYS = {"ann_meta": ["announcement_id"],
"news_meta": ["art_code", "stock_code"],
@@ -196,7 +197,7 @@ def load_stock_pool(refresh=False):
j = json.loads(cache.read_text(encoding="utf-8"))
return sorted(j.items())
client = DomainClient("cninfo")
r = client.request("GET", POOL_URL)
r = client.request("GET", POOL_URL, headers=UA)
pairs = [(it["code"], it["orgId"])
for it in r.json().get("stockList") or []]
pairs = filter_stock_pool(pairs)
@@ -424,7 +425,7 @@ def fetch_cninfo_announcements(client, code, org_id, start, end, category=""):
out, page = [], 1
while True:
data["pageNum"] = str(page)
r = client.request("POST", QUERY_URL, data=data)
r = client.request("POST", QUERY_URL, data=data, headers=UA)
try:
j = r.json()
except ValueError as e:
@@ -446,24 +447,26 @@ def fetch_news_recent(client, code):
"postTag": "</em>"}}}
params = {"cb": "cb", "param": json.dumps(inner, ensure_ascii=False),
"_": str(int(time.time() * 1000))}
r = client.request("GET", SEARCH_API_URL, params=params)
r = client.request("GET", SEARCH_API_URL, params=params, headers=UA)
s = r.text.strip()
if "(" not in s or not s.endswith(")"):
raise TransportError(f"search-api 非 jsonp 响应: {s[:120]!r}")
j = json.loads(s[s.index("(") + 1:s.rindex(")")])
return (j.get("result") or {}).get("cmsArticleWebOld") or []
def fetch_news_backfill_page(client, code, mkt_prefix, page):
url = NP_LIST_URL.format(mkt=mkt_prefix, code=code, page=page)
r = client.request("GET", url)
r = client.request("GET", url, headers=UA)
return (r.json().get("data") or {}).get("list") or []
def fetch_article_html(client, url):
return client.request("GET", url).text
return client.request("GET", url, headers=UA).text
def download_pdf(client, url, dest):
r = client.request("GET", url)
r = client.request("GET", url, headers=UA)
dest.parent.mkdir(parents=True, exist_ok=True)
tmp = dest.with_suffix(".tmp")
tmp.write_bytes(r.content)