sanguo/sanguo_vnpy_v2
2
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(data): 中证1000/2000 历史成份股补全(parse+migrate 聚合)

Browse Source 
parse_csindex_announce.py 3 处修复:
- 932000 launch xlsx header 定位"证券代码"列(原 row[0]=指数代码 bug, distinct=1)
- 000852 列表搜索用 indexCode payload 精准拉(28 -> 96 公告, 回溯到 2016)
- akshare current snapshot header 定位"成分券代码"列(原 iloc[0]=日期 bug, current=0)

migrate_constituent.py:
- 加 _read_announce_union_aggregated(): 000852/932000 用 announce_union 全集
  替换原 snapshot-only 路径, was_removed 治幸存者偏差
- 缺 announce_union 时回退旧 snapshot 逻辑(向后兼容)

TDD: tests/portfolio/test_migrate_announce_union.py 7 测全过(5 unit + 2 integration)
Mac 产出验证:
- 000852: distinct 1220 -> 1672, was_removed 0 -> 672
- 932000: distinct 1 -> 2684(launch 修复), was_removed 0 -> 684
This commit is contained in:
Claude Dev
2026-07-23 23:45:55 +08:00
parent 114a69e997
commit b237c2d7e7
3 changed files with 448 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/data_platform/migrate_constituent.py
+82 -16
View File
@@ -77,6 +77,61 @@ def _read_baostock_constituent(c: sqlite3.Connection) -> pd.DataFrame:
return df
def _read_announce_union_aggregated(hist_dir: str,
index_codes: list[str]) -> pd.DataFrame:
"""读 *_announce_union.parquet, 聚合成全集型(index_code/code/code_name/
in_current/was_removed/source)。
announce_union schema:
updateDate / index_code / code / code_name / adjust_type / notice_id / source
adjust_type: add | remove | current | initial
全集聚合(spec §14):
ever_codes = announce_union 所有 distinct code(任意 adjust_type)
current_codes = announce_union 中 adjust_type='current' 的 code
(∪ <idx>_snapshot.parquet 兜底, 兼容旧 announce 无 current 行)
in_current = code in current_codes
was_removed = not in_current(曾经入选已踢)
code_name = 优先 snapshot 当前名(announce 的历史名可能过时)
缺 announce_union 文件 -> 返回空 DataFrame(上层走 snapshot 兜底)。
"""
cols = ["index_code", "code", "code_name", "in_current", "was_removed", "source"]
rows = []
for idx in index_codes:
ann_path = os.path.join(hist_dir, f"{idx}_announce_union.parquet")
if not os.path.exists(ann_path):
continue
ann = pd.read_parquet(ann_path)
ann["code"] = ann["code"].apply(norm_code)
# current 兜底: announce 内 current 行 + snapshot 文件
current_codes = set(ann.loc[ann["adjust_type"] == "current", "code"])
snap_path = os.path.join(hist_dir, f"{idx}_snapshot.parquet")
snap_name_map: dict[str, str] = {}
if os.path.exists(snap_path):
snap = pd.read_parquet(snap_path)
snap["code"] = snap["code"].apply(norm_code)
current_codes |= set(snap["code"])
snap_name_map = dict(zip(snap["code"], snap["code_name"]))
# ever = distinct(code), name 取 announce 或 snapshot(优先 snapshot)
ever = (
ann[["code", "code_name"]]
.drop_duplicates(subset=["code"])
)
for _, r in ever.iterrows():
code = r["code"]
in_cur = code in current_codes
name = snap_name_map.get(code) or r["code_name"]
rows.append({
"index_code": idx, "code": code, "code_name": name,
"in_current": int(in_cur), "was_removed": int(not in_cur),
"source": "csindex_announce",
})
if not rows:
return pd.DataFrame(columns=cols)
return pd.DataFrame(rows, columns=cols)
def migrate(db_path: str) -> None:
"""(幂等)重建 constituent_unified_staging。
@@ -115,23 +170,30 @@ def migrate(db_path: str) -> None:
df_deep["code"] = df_deep["code"].apply(norm_code)
print(f"[深证 union] rows={len(df_deep)}")
# 3. 中证 snapshot
snap = []
for f in [os.path.join(HIST, "000852_snapshot.parquet"),
os.path.join(HIST, "932000_snapshot.parquet")]:
if os.path.exists(f):
d = pd.read_parquet(f)[["index_code", "code", "code_name"]]
d["in_current"] = True
d["was_removed"] = False
d["source"] = "akshare_csindex"
snap.append(d)
df_snap = pd.concat(snap, ignore_index=True) if snap else pd.DataFrame(
columns=["index_code", "code", "code_name", "in_current", "was_removed", "source"])
df_snap["code"] = df_snap["code"].apply(norm_code)
print(f"[中证 snapshot] rows={len(df_snap)}")
# 3. 中证 1000/2000 announce_union 全集(优先); 缺则回退 snapshot 当前
ANNOUNCE_IDX = ["000852", "932000"]
df_csi = _read_announce_union_aggregated(HIST, ANNOUNCE_IDX)
if df_csi.empty:
# 回退: 旧 snapshot-only 路径(announce_union 未生成时)
snap = []
for f in [os.path.join(HIST, "000852_snapshot.parquet"),
os.path.join(HIST, "932000_snapshot.parquet")]:
if os.path.exists(f):
d = pd.read_parquet(f)[["index_code", "code", "code_name"]]
d["in_current"] = True
d["was_removed"] = False
d["source"] = "akshare_csindex"
snap.append(d)
df_csi = pd.concat(snap, ignore_index=True) if snap else pd.DataFrame(
columns=["index_code", "code", "code_name", "in_current", "was_removed", "source"])
df_csi["code"] = df_csi["code"].apply(norm_code)
print(f"[中证 snapshot 回退] rows={len(df_csi)}")
else:
print(f"[中证 announce_union] rows={len(df_csi)}, "
f"indices={df_csi['index_code'].nunique()}")
# 合并 + 去重 (同 index+code+source)
all_df = pd.concat([pool, df_deep, df_snap], ignore_index=True)
all_df = pd.concat([pool, df_deep, df_csi], ignore_index=True)
all_df = all_df.drop_duplicates(["index_code", "code", "source"])
print(f"\n[TOTAL] constituent_unified: {len(all_df)} rows, "
f"{all_df['index_code'].nunique()} indices")
@@ -163,7 +225,11 @@ def migrate(db_path: str) -> None:
"SELECT COUNT(*), SUM(in_current), SUM(was_removed) FROM "
f"{STAGING} WHERE index_code='399001'").fetchone())
print("sample 000852:", c.execute(
"SELECT COUNT(*) FROM " f"{STAGING} WHERE index_code='000852'").fetchone())
"SELECT COUNT(*), SUM(in_current), SUM(was_removed) FROM "
f"{STAGING} WHERE index_code='000852'").fetchone())
print("sample 932000:", c.execute(
"SELECT COUNT(*), SUM(in_current), SUM(was_removed) FROM "
f"{STAGING} WHERE index_code='932000'").fetchone())
finally:
c.close()
print("\nMIGRATE STAGING DONE (未 rename, 验证 OK 后单独合并)")