fix(data): corpus 全 fetcher 补 UA+jsonp 解析防御 [nas]
NAS 冒烟二跑实锤: search-api 对无 UA 请求返 200+空 body(金丝雀当时带 UA 故
未暴露)→s.index('(') ValueError 裸崩。修: 统一 UA 常量六处 fetcher 全带;
jsonp 非 cb(...) 形态 → TransportError(带响应片段,计入域名连续错走冷却
而非裸崩);容器内 A/B 实测 no-UA='' vs withUA=正常610条
This commit is contained in:
@@ -78,6 +78,7 @@ PDF_CAPACITY_CHECK_GB = 50 # 每落 50GB 自查一次 df
|
|||||||
|
|
||||||
RATE = {"cninfo": 1.0, "eastmoney": 1.0, "article": 2.0} # 每域最小间隔(秒), 慢爬纪律
|
RATE = {"cninfo": 1.0, "eastmoney": 1.0, "article": 2.0} # 每域最小间隔(秒), 慢爬纪律
|
||||||
JITTER = (0.0, 0.3)
|
JITTER = (0.0, 0.3)
|
||||||
|
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
|
||||||
|
|
||||||
ID_KEYS = {"ann_meta": ["announcement_id"],
|
ID_KEYS = {"ann_meta": ["announcement_id"],
|
||||||
"news_meta": ["art_code", "stock_code"],
|
"news_meta": ["art_code", "stock_code"],
|
||||||
@@ -196,7 +197,7 @@ def load_stock_pool(refresh=False):
|
|||||||
j = json.loads(cache.read_text(encoding="utf-8"))
|
j = json.loads(cache.read_text(encoding="utf-8"))
|
||||||
return sorted(j.items())
|
return sorted(j.items())
|
||||||
client = DomainClient("cninfo")
|
client = DomainClient("cninfo")
|
||||||
r = client.request("GET", POOL_URL)
|
r = client.request("GET", POOL_URL, headers=UA)
|
||||||
pairs = [(it["code"], it["orgId"])
|
pairs = [(it["code"], it["orgId"])
|
||||||
for it in r.json().get("stockList") or []]
|
for it in r.json().get("stockList") or []]
|
||||||
pairs = filter_stock_pool(pairs)
|
pairs = filter_stock_pool(pairs)
|
||||||
@@ -424,7 +425,7 @@ def fetch_cninfo_announcements(client, code, org_id, start, end, category=""):
|
|||||||
out, page = [], 1
|
out, page = [], 1
|
||||||
while True:
|
while True:
|
||||||
data["pageNum"] = str(page)
|
data["pageNum"] = str(page)
|
||||||
r = client.request("POST", QUERY_URL, data=data)
|
r = client.request("POST", QUERY_URL, data=data, headers=UA)
|
||||||
try:
|
try:
|
||||||
j = r.json()
|
j = r.json()
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
@@ -446,24 +447,26 @@ def fetch_news_recent(client, code):
|
|||||||
"postTag": "</em>"}}}
|
"postTag": "</em>"}}}
|
||||||
params = {"cb": "cb", "param": json.dumps(inner, ensure_ascii=False),
|
params = {"cb": "cb", "param": json.dumps(inner, ensure_ascii=False),
|
||||||
"_": str(int(time.time() * 1000))}
|
"_": str(int(time.time() * 1000))}
|
||||||
r = client.request("GET", SEARCH_API_URL, params=params)
|
r = client.request("GET", SEARCH_API_URL, params=params, headers=UA)
|
||||||
s = r.text.strip()
|
s = r.text.strip()
|
||||||
|
if "(" not in s or not s.endswith(")"):
|
||||||
|
raise TransportError(f"search-api 非 jsonp 响应: {s[:120]!r}")
|
||||||
j = json.loads(s[s.index("(") + 1:s.rindex(")")])
|
j = json.loads(s[s.index("(") + 1:s.rindex(")")])
|
||||||
return (j.get("result") or {}).get("cmsArticleWebOld") or []
|
return (j.get("result") or {}).get("cmsArticleWebOld") or []
|
||||||
|
|
||||||
|
|
||||||
def fetch_news_backfill_page(client, code, mkt_prefix, page):
|
def fetch_news_backfill_page(client, code, mkt_prefix, page):
|
||||||
url = NP_LIST_URL.format(mkt=mkt_prefix, code=code, page=page)
|
url = NP_LIST_URL.format(mkt=mkt_prefix, code=code, page=page)
|
||||||
r = client.request("GET", url)
|
r = client.request("GET", url, headers=UA)
|
||||||
return (r.json().get("data") or {}).get("list") or []
|
return (r.json().get("data") or {}).get("list") or []
|
||||||
|
|
||||||
|
|
||||||
def fetch_article_html(client, url):
|
def fetch_article_html(client, url):
|
||||||
return client.request("GET", url).text
|
return client.request("GET", url, headers=UA).text
|
||||||
|
|
||||||
|
|
||||||
def download_pdf(client, url, dest):
|
def download_pdf(client, url, dest):
|
||||||
r = client.request("GET", url)
|
r = client.request("GET", url, headers=UA)
|
||||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||||
tmp = dest.with_suffix(".tmp")
|
tmp = dest.with_suffix(".tmp")
|
||||||
tmp.write_bytes(r.content)
|
tmp.write_bytes(r.content)
|
||||||
|
|||||||
Reference in New Issue
Block a user