From ae6e2d92d01e12951bc657eb554de591d7874653 Mon Sep 17 00:00:00 2001
From: claude_dev <claude-dev@sanguo.local>
Date: Sun, 6 Sep 2026 23:44:48 +0800
Subject: [PATCH] =?UTF-8?q?fix(data):=20corpus=20=E5=85=A8=20fetcher=20?=
 =?UTF-8?q?=E8=A1=A5=20UA+jsonp=20=E8=A7=A3=E6=9E=90=E9=98=B2=E5=BE=A1=20[?=
 =?UTF-8?q?nas]?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

NAS 冒烟二跑实锤: search-api 对无 UA 请求返 200+空 body(金丝雀当时带 UA 故
未暴露)→s.index('(') ValueError 裸崩。修: 统一 UA 常量六处 fetcher 全带;
jsonp 非 cb(...) 形态 → TransportError(带响应片段,计入域名连续错走冷却
而非裸崩);容器内 A/B 实测 no-UA='' vs withUA=正常610条
---
 scripts/data_platform/corpus_download.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/scripts/data_platform/corpus_download.py b/scripts/data_platform/corpus_download.py
index 8c5ccfe..bea3956 100644
--- a/scripts/data_platform/corpus_download.py
+++ b/scripts/data_platform/corpus_download.py
@@ -78,6 +78,7 @@ PDF_CAPACITY_CHECK_GB = 50      # 每落 50GB 自查一次 df
 
 RATE = {"cninfo": 1.0, "eastmoney": 1.0, "article": 2.0}   # 每域最小间隔(秒), 慢爬纪律
 JITTER = (0.0, 0.3)
+UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
 
 ID_KEYS = {"ann_meta": ["announcement_id"],
            "news_meta": ["art_code", "stock_code"],
@@ -196,7 +197,7 @@ def load_stock_pool(refresh=False):
             j = json.loads(cache.read_text(encoding="utf-8"))
             return sorted(j.items())
     client = DomainClient("cninfo")
-    r = client.request("GET", POOL_URL)
+    r = client.request("GET", POOL_URL, headers=UA)
     pairs = [(it["code"], it["orgId"])
              for it in r.json().get("stockList") or []]
     pairs = filter_stock_pool(pairs)
@@ -424,7 +425,7 @@ def fetch_cninfo_announcements(client, code, org_id, start, end, category=""):
     out, page = [], 1
     while True:
         data["pageNum"] = str(page)
-        r = client.request("POST", QUERY_URL, data=data)
+        r = client.request("POST", QUERY_URL, data=data, headers=UA)
         try:
             j = r.json()
         except ValueError as e:
@@ -446,24 +447,26 @@ def fetch_news_recent(client, code):
                                             "postTag": "</em>"}}}
     params = {"cb": "cb", "param": json.dumps(inner, ensure_ascii=False),
               "_": str(int(time.time() * 1000))}
-    r = client.request("GET", SEARCH_API_URL, params=params)
+    r = client.request("GET", SEARCH_API_URL, params=params, headers=UA)
     s = r.text.strip()
+    if "(" not in s or not s.endswith(")"):
+        raise TransportError(f"search-api 非 jsonp 响应: {s[:120]!r}")
     j = json.loads(s[s.index("(") + 1:s.rindex(")")])
     return (j.get("result") or {}).get("cmsArticleWebOld") or []
 
 
 def fetch_news_backfill_page(client, code, mkt_prefix, page):
     url = NP_LIST_URL.format(mkt=mkt_prefix, code=code, page=page)
-    r = client.request("GET", url)
+    r = client.request("GET", url, headers=UA)
     return (r.json().get("data") or {}).get("list") or []
 
 
 def fetch_article_html(client, url):
-    return client.request("GET", url).text
+    return client.request("GET", url, headers=UA).text
 
 
 def download_pdf(client, url, dest):
-    r = client.request("GET", url)
+    r = client.request("GET", url, headers=UA)
     dest.parent.mkdir(parents=True, exist_ok=True)
     tmp = dest.with_suffix(".tmp")
     tmp.write_bytes(r.content)