fix(data): corpus 全 fetcher 补 UA+jsonp 解析防御 [nas]
NAS 冒烟二跑实锤: search-api 对无 UA 请求返 200+空 body(金丝雀当时带 UA 故
未暴露)→s.index('(') ValueError 裸崩。修: 统一 UA 常量六处 fetcher 全带;
jsonp 非 cb(...) 形态 → TransportError(带响应片段,计入域名连续错走冷却
而非裸崩);容器内 A/B 实测 no-UA='' vs withUA=正常610条
This commit is contained in:
@@ -78,6 +78,7 @@ PDF_CAPACITY_CHECK_GB = 50 # 每落 50GB 自查一次 df
|
||||
|
||||
RATE = {"cninfo": 1.0, "eastmoney": 1.0, "article": 2.0} # 每域最小间隔(秒), 慢爬纪律
|
||||
JITTER = (0.0, 0.3)
|
||||
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
|
||||
|
||||
ID_KEYS = {"ann_meta": ["announcement_id"],
|
||||
"news_meta": ["art_code", "stock_code"],
|
||||
@@ -196,7 +197,7 @@ def load_stock_pool(refresh=False):
|
||||
j = json.loads(cache.read_text(encoding="utf-8"))
|
||||
return sorted(j.items())
|
||||
client = DomainClient("cninfo")
|
||||
r = client.request("GET", POOL_URL)
|
||||
r = client.request("GET", POOL_URL, headers=UA)
|
||||
pairs = [(it["code"], it["orgId"])
|
||||
for it in r.json().get("stockList") or []]
|
||||
pairs = filter_stock_pool(pairs)
|
||||
@@ -424,7 +425,7 @@ def fetch_cninfo_announcements(client, code, org_id, start, end, category=""):
|
||||
out, page = [], 1
|
||||
while True:
|
||||
data["pageNum"] = str(page)
|
||||
r = client.request("POST", QUERY_URL, data=data)
|
||||
r = client.request("POST", QUERY_URL, data=data, headers=UA)
|
||||
try:
|
||||
j = r.json()
|
||||
except ValueError as e:
|
||||
@@ -446,24 +447,26 @@ def fetch_news_recent(client, code):
|
||||
"postTag": "</em>"}}}
|
||||
params = {"cb": "cb", "param": json.dumps(inner, ensure_ascii=False),
|
||||
"_": str(int(time.time() * 1000))}
|
||||
r = client.request("GET", SEARCH_API_URL, params=params)
|
||||
r = client.request("GET", SEARCH_API_URL, params=params, headers=UA)
|
||||
s = r.text.strip()
|
||||
if "(" not in s or not s.endswith(")"):
|
||||
raise TransportError(f"search-api 非 jsonp 响应: {s[:120]!r}")
|
||||
j = json.loads(s[s.index("(") + 1:s.rindex(")")])
|
||||
return (j.get("result") or {}).get("cmsArticleWebOld") or []
|
||||
|
||||
|
||||
def fetch_news_backfill_page(client, code, mkt_prefix, page):
|
||||
url = NP_LIST_URL.format(mkt=mkt_prefix, code=code, page=page)
|
||||
r = client.request("GET", url)
|
||||
r = client.request("GET", url, headers=UA)
|
||||
return (r.json().get("data") or {}).get("list") or []
|
||||
|
||||
|
||||
def fetch_article_html(client, url):
|
||||
return client.request("GET", url).text
|
||||
return client.request("GET", url, headers=UA).text
|
||||
|
||||
|
||||
def download_pdf(client, url, dest):
|
||||
r = client.request("GET", url)
|
||||
r = client.request("GET", url, headers=UA)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = dest.with_suffix(".tmp")
|
||||
tmp.write_bytes(r.content)
|
||||
|
||||
Reference in New Issue
Block a user