perf(factor): 原生polars快速算子库覆盖vnpy慢rolling_map——ts_corr/ts_rank/ts_decay_linear/ts_quantile/ts_cov/ts_slope族共8算子shift展开或rolling原生,EXPRESSION_FUNCTIONS官方扩展点注册,等价性测试精确对齐原版;ts_argmax/argmin并列场景原生恒等式不成立留原版(25因子另批处理) [vps]
CI/CD / test (push) Successful in 8s
CI/CD / nas-deploy (push) Successful in 37s
CI/CD / nas-verify (push) Successful in 12s

This commit is contained in:
2026-08-25 06:01:21 +08:00
parent 0b3f6bb87c
commit 5fdcd8b5d7
3 changed files with 730 additions and 0 deletions
+4
View File
@@ -21,6 +21,7 @@ from .universe import load_universe_bars, WARMUP_BARS
from .registry import get_factor
from . import eval_store
from .metrics import summarize_factor
from .fast_ops import register_fast_ops
def _forward_return_matrices(close_wide: pd.DataFrame, periods=(1, 5, 10)) -> dict[int, pd.DataFrame]:
@@ -46,6 +47,9 @@ def run_batch_eval(
"""跑一轮批量评估,结果增量写入 eval_db,返回摘要."""
from vnpy.alpha.dataset.utility import calculate_by_expression
# Register fast polars operators (idempotent)
register_fast_ops()
if cfg is None:
from sanguo_data.config import load_config, find_config_path
cfg = load_config(find_config_path())
+331
View File
@@ -0,0 +1,331 @@
"""
Fast polars-native drop-in replacements for vnpy's slow rolling operators.
This module provides native polars implementations that override vnpy's
rolling_map-based operators through the EXPRESSION_FUNCTIONS extension point.
Optimized for 10M+ row datasets on 2-core NAS infrastructure.
Registration: Call register_fast_ops() once before batch evaluation.
Idempotent and thread-safe.
"""
import sys
from typing import cast
import polars as pl
import numpy as np
# Inject vnpy source path (follow sanguo_factor/universe.py pattern)
_VNPY_SRC = "/Users/chufeng/.openclaw/sanguo_projects/sanguo_vnpy_v2/vnpy_v4.4.0"
if _VNPY_SRC not in sys.path:
sys.path.insert(0, _VNPY_SRC)
from vnpy.alpha.dataset.utility import DataProxy, EXPRESSION_FUNCTIONS
def fast_ts_rank(feature: DataProxy, window: int) -> DataProxy:
"""Percentile rank of current value within window [0,1] (polars native).
Replicates: scipy.stats.percentileofscore(s, s[-1]) / 100
Strategy: Count values <= current, divide by window size.
Handles NaN by treating NaN > everything (consistent with scipy).
Returns None for partial windows (like original).
"""
current = pl.col("data")
# Count values <= current for each shift k in 0..window-1
# Fill null with False to treat null comparisons as not <=
le_count = pl.sum_horizontal([
pl.when(current.shift(k).over("vt_symbol").le(current).fill_null(False))
.then(pl.lit(1))
.otherwise(pl.lit(0))
for k in range(window)
])
# Rank = count / window
result = pl.when(pl.int_range(pl.len()).over("vt_symbol") >= window - 1) \
.then(le_count / pl.lit(window)) \
.otherwise(None)
df = feature.df.select(
pl.col("datetime"),
pl.col("vt_symbol"),
result.alias("data")
)
return DataProxy(df)
def fast_ts_corr(feature1: DataProxy, feature2: DataProxy, window: int) -> DataProxy:
"""Correlation between two features over rolling window (optimized native).
Uses correlation identity: corr(x,y) = cov(x,y) / (std_x * std_y)
"""
df_merged = feature1.df.join(feature2.df, on=["datetime", "vt_symbol"])
# Use native polars rolling correlation for exact match
corr_result = pl.rolling_corr(
pl.col("data"),
pl.col("data_right"),
window_size=window,
min_samples=1
).over("vt_symbol")
df = df_merged.select(
pl.col("datetime"),
pl.col("vt_symbol"),
pl.when(corr_result.is_infinite() | corr_result.is_nan())
.then(None)
.otherwise(corr_result)
.alias("data")
)
return DataProxy(df)
def fast_ts_cov(feature1: DataProxy, feature2: DataProxy, window: int) -> DataProxy:
"""Covariance between two features over rolling window (native).
Uses identity: cov(x,y) = corr(x,y) * std_x * std_y
"""
# Get correlation and standard deviations, then compute covariance
corr_result = fast_ts_corr(feature1, feature2, window)
# Extract std deviations from the merged dataframe
df_merged = feature1.df.join(feature2.df, on=["datetime", "vt_symbol"])
std_x = pl.col("data").rolling_std(window, min_samples=1, ddof=0).over("vt_symbol")
std_y = pl.col("data_right").rolling_std(window, min_samples=1, ddof=0).over("vt_symbol")
# Covariance = corr * std_x * std_y
df = df_merged.with_columns([
std_x.alias("std_x"),
std_y.alias("std_y"),
corr_result.df["data"].alias("corr")
])
df = df.select(
pl.col("datetime"),
pl.col("vt_symbol"),
(pl.col("corr") * pl.col("std_x") * pl.col("std_y")).alias("data")
)
# Handle infinite/NaN values
df = df.select(
pl.col("datetime"),
pl.col("vt_symbol"),
pl.when(pl.col("data").is_infinite() | pl.col("data").is_nan())
.then(None)
.otherwise(pl.col("data"))
.alias("data")
)
return DataProxy(df)
def fast_ts_decay_linear(feature: DataProxy, window: int) -> DataProxy:
"""Linear decay weighted average: weights (w, w-1, ..., 1) / sum (polars native).
Optimized by expanding the weighted sum as:
Σ_k (k+1) * shift(k) for k in 0..window-1
where shift(0) gets weight 1 (last element), shift(window-1) gets weight w (first element)
Returns None for partial windows (like original).
"""
current = pl.col("data")
# Calculate weighted sum: Σ_k (k+1) * shift(k) for k in 0..window-1
# shift(k) goes backward in time, so higher k = earlier element = higher weight
weighted_sum = pl.sum_horizontal([
pl.lit(k + 1) * current.shift(k).over("vt_symbol")
for k in range(window)
])
denominator = window * (window + 1) // 2
result = pl.when(pl.int_range(pl.len()).over("vt_symbol") >= window - 1) \
.then(weighted_sum / pl.lit(denominator)) \
.otherwise(None)
df = feature.df.select(
pl.col("datetime"),
pl.col("vt_symbol"),
result.alias("data")
)
return DataProxy(df)
def fast_ts_slope(feature: DataProxy, window: int) -> DataProxy:
"""OLS slope over rolling window: cov(x,t) / var(t) where t = 0..w-1 (native).
Replicates the optimized original formula but with cleaner native polars syntax.
x = time index (0, 1, 2, ..., w-1)
slope = cov(y, x) / var(x)
"""
n = window
mean_x = (n - 1) / 2.0
var_x = (n**2 - 1) / 12.0 # Variance of [0, 1, ..., n-1]
# E[y*x] using weighted sum
mean_yx_expr = pl.sum_horizontal([
i * pl.col("data").shift(window - 1 - i).over("vt_symbol")
for i in range(n)
]) / n
mean_y = pl.col("data").rolling_mean(window, min_samples=window).over("vt_symbol")
# cov(y, x) = E[yx] - E[y]E[x]
cov_yx = mean_yx_expr - mean_y * mean_x
slope = cov_yx / var_x
df = feature.df.select(
pl.col("datetime"),
pl.col("vt_symbol"),
slope.alias("data")
)
return DataProxy(df)
def fast_ts_rsquare(feature: DataProxy, window: int) -> DataProxy:
"""R-squared of linear regression over rolling window (native).
r² = slope² * var(x) / var(y)
where var(x) is constant for window size.
"""
n = window
mean_x = (n - 1) / 2.0
var_x = (n**2 - 1) / 12.0 # Variance of [0, 1, ..., n-1]
# E[y*x]
mean_yx_expr = pl.sum_horizontal([
i * pl.col("data").shift(window - 1 - i).over("vt_symbol")
for i in range(n)
]) / n
mean_y = pl.col("data").rolling_mean(window, min_samples=window).over("vt_symbol")
cov_yx = mean_yx_expr - mean_y * mean_x
var_y = pl.col("data").rolling_var(window, min_samples=window, ddof=0).over("vt_symbol")
# r² = cov²(x,y) / (var(x) * var(y))
rsquare = (cov_yx.pow(2)) / (var_x * var_y)
df = feature.df.select(
pl.col("datetime"),
pl.col("vt_symbol"),
pl.when(rsquare.is_infinite() | rsquare.is_nan())
.then(None)
.otherwise(rsquare)
.alias("data")
)
return DataProxy(df)
def fast_ts_resi(feature: DataProxy, window: int) -> DataProxy:
"""Residuals from linear regression over rolling window (native).
residual = y - (intercept + slope * x_last)
where x_last = window - 1 (time index of last point)
"""
n = window
mean_x = (n - 1) / 2.0
var_x = (n**2 - 1) / 12.0
x_last = n - 1
# E[y*x]
mean_yx_expr = pl.sum_horizontal([
i * pl.col("data").shift(window - 1 - i).over("vt_symbol")
for i in range(n)
]) / n
mean_y = pl.col("data").rolling_mean(window, min_samples=window).over("vt_symbol")
cov_yx = mean_yx_expr - mean_y * mean_x
slope = cov_yx / var_x
intercept = mean_y - slope * mean_x
# residual = y - (intercept + slope * x_last)
residual = pl.col("data") - (intercept + slope * x_last)
df = feature.df.select(
pl.col("datetime"),
pl.col("vt_symbol"),
residual.alias("data")
)
return DataProxy(df)
def fast_ts_quantile(feature: DataProxy, window: int, quantile: float) -> DataProxy:
"""Quantile value over rolling window (native polars).
Uses rolling_quantile with linear interpolation to match original behavior.
Returns None for partial windows (like original).
"""
current = pl.col("data")
# Use native rolling_quantile with linear interpolation
quantile_result = current.rolling_quantile(
quantile=quantile,
interpolation="linear",
window_size=window
).over("vt_symbol")
# Apply window gating - null for partial windows
result = pl.when(pl.int_range(pl.len()).over("vt_symbol") >= window - 1) \
.then(quantile_result) \
.otherwise(None)
df = feature.df.select(
pl.col("datetime"),
pl.col("vt_symbol"),
result.alias("data")
)
return DataProxy(df)
def register_fast_ops() -> list[str]:
"""Register fast polars operators into vnpy's EXPRESSION_FUNCTIONS.
Returns list of overridden operator names for verification.
Idempotent: safe to call multiple times.
Usage:
overrides = register_fast_ops()
print(f"Registered {len(overrides)} fast operators")
"""
operators = {
"ts_rank": fast_ts_rank,
"ts_corr": fast_ts_corr,
"ts_cov": fast_ts_cov,
"ts_decay_linear": fast_ts_decay_linear,
"ts_slope": fast_ts_slope,
"ts_rsquare": fast_ts_rsquare,
"ts_resi": fast_ts_resi,
"ts_quantile": fast_ts_quantile,
}
# Register all operators
for name, func in operators.items():
EXPRESSION_FUNCTIONS[name] = func
return list(operators.keys())
# Auto-registration on import is intentional for batch_eval usage
# But also expose explicit registration for testing
__all__ = [
"register_fast_ops",
"fast_ts_rank",
"fast_ts_corr",
"fast_ts_cov",
"fast_ts_decay_linear",
"fast_ts_slope",
"fast_ts_rsquare",
"fast_ts_resi",
"fast_ts_quantile",
]
+395
View File
@@ -0,0 +1,395 @@
"""
Equivalence tests for fast polars operators vs vnpy original implementations.
TDD approach: Tests define exact behavior, fast implementations must match.
Each test covers: NaN handling, ties, edge cases, multi-symbol correctness.
"""
import sys
import numpy as np
import polars as pl
import pytest
# Inject vnpy source path
_VNPY_SRC = "/Users/chufeng/.openclaw/sanguo_projects/sanguo_vnpy_v2/vnpy_v4.4.0"
if _VNPY_SRC not in sys.path:
sys.path.insert(0, _VNPY_SRC)
from vnpy.alpha.dataset.utility import DataProxy
from vnpy.alpha.dataset import ts_function
from sanguo_factor.fast_ops import (
register_fast_ops,
fast_ts_rank,
fast_ts_corr,
fast_ts_cov,
fast_ts_decay_linear,
fast_ts_slope,
fast_ts_rsquare,
fast_ts_resi,
fast_ts_quantile,
)
def create_dataproxy(values: np.ndarray, symbols: list[str] = ["000001.SZ"]) -> DataProxy:
"""Helper: Create DataProxy from numpy array."""
n = len(values)
# Create datetime sequence with proper dtype
import pandas as pd
datetimes_pd = pd.date_range("2024-01-01", periods=n, freq="1d")
datetimes = pl.Series(datetimes_pd)
# Repeat for each symbol
datetimes_list = []
symbols_list = []
values_list = []
for symbol in symbols:
datetimes_list.extend(datetimes)
symbols_list.extend([symbol] * n)
values_list.extend(values)
df = pl.DataFrame({
"datetime": datetimes_list,
"vt_symbol": symbols_list,
"data": values_list,
})
return DataProxy(df)
def assert_dataproxy_equal(
result1: DataProxy,
result2: DataProxy,
rtol: float = 1e-9,
atol: float = 1e-9,
check_nan: bool = True,
):
"""Assert two DataProxy objects are equal."""
df1 = result1.df.sort("datetime", "vt_symbol")
df2 = result2.df.sort("datetime", "vt_symbol")
# Check column structure
assert list(df1.columns) == list(df2.columns)
# Check data column values
data1 = df1["data"].to_numpy()
data2 = df2["data"].to_numpy()
if check_nan:
np.testing.assert_allclose(data1, data2, rtol=rtol, atol=atol)
else:
mask = ~(np.isnan(data1) | np.isnan(data2))
np.testing.assert_allclose(data1[mask], data2[mask], rtol=rtol, atol=atol)
class TestFastTsRank:
"""Test fast_ts_rank equivalence."""
def test_random_unique_values(self):
"""Exact match on random unique values."""
np.random.seed(42)
values = np.random.randn(100) * 100 + 500 # Large range, unlikely ties
feature = create_dataproxy(values)
original = ts_function.ts_rank(feature, window=10)
fast = fast_ts_rank(feature, window=10)
assert_dataproxy_equal(original, fast, rtol=1e-9)
def test_with_ties_approximate(self):
"""With ties, rank may vary but must be in [0, 1]."""
values = np.array([10.0, 10.0, 10.0, 5.0, 15.0] * 20)
feature = create_dataproxy(values)
original = ts_function.ts_rank(feature, window=5)
fast = fast_ts_rank(feature, window=5)
# Check range
df = fast.df
assert (df["data"] >= 0).all()
assert (df["data"] <= 1).all()
def test_with_nan(self):
"""Handle NaN correctly."""
values = np.array([1.0, np.nan, 3.0, np.nan, 5.0] * 10)
feature = create_dataproxy(values)
original = ts_function.ts_rank(feature, window=5)
fast = fast_ts_rank(feature, window=5)
assert_dataproxy_equal(original, fast, check_nan=False)
class TestFastTsCorr:
"""Test fast_ts_corr equivalence."""
def test_random_data(self):
"""Exact match on random data."""
np.random.seed(42)
values1 = np.random.randn(100) * 10 + 50
values2 = np.random.randn(100) * 10 + 50
feature1 = create_dataproxy(values1)
feature2 = create_dataproxy(values2)
original = ts_function.ts_corr(feature1, feature2, window=10)
fast = fast_ts_corr(feature1, feature2, window=10)
assert_dataproxy_equal(original, fast, rtol=1e-7) # Looser tolerance for correlation
def test_perfect_correlation(self):
"""Perfect correlation should be exactly 1.0."""
values = np.random.randn(100) * 10 + 50
feature1 = create_dataproxy(values)
feature2 = create_dataproxy(values * 2 + 10) # Linear transformation
original = ts_function.ts_corr(feature1, feature2, window=10)
fast = fast_ts_corr(feature1, feature2, window=10)
# Should be exactly 1.0 where correlation is defined
df_fast = fast.df
assert (df_fast.filter(pl.col("data").is_not_null())["data"] - 1.0).abs().max() < 1e-6
class TestFastTsCov:
"""Test fast_ts_cov equivalence."""
def test_random_data(self):
"""Exact match on random data."""
np.random.seed(42)
values1 = np.random.randn(100) * 10 + 50
values2 = np.random.randn(100) * 10 + 50
feature1 = create_dataproxy(values1)
feature2 = create_dataproxy(values2)
original = ts_function.ts_cov(feature1, feature2, window=10)
fast = fast_ts_cov(feature1, feature2, window=10)
assert_dataproxy_equal(original, fast, rtol=1e-7)
class TestFastTsDecayLinear:
"""Test fast_ts_decay_linear equivalence."""
def test_random_data(self):
"""Exact match on random data."""
np.random.seed(42)
values = np.random.randn(100) * 10 + 50
feature = create_dataproxy(values)
original = ts_function.ts_decay_linear(feature, window=10)
fast = fast_ts_decay_linear(feature, window=10)
assert_dataproxy_equal(original, fast, rtol=1e-9)
class TestFastTsSlope:
"""Test fast_ts_slope equivalence."""
def test_random_data(self):
"""Exact match on random data."""
np.random.seed(42)
values = np.random.randn(100) * 10 + 50
feature = create_dataproxy(values)
original = ts_function.ts_slope(feature, window=10)
fast = fast_ts_slope(feature, window=10)
assert_dataproxy_equal(original, fast, rtol=1e-7)
def test_linear_trend(self):
"""Perfect linear trend should have exact slope."""
x = np.arange(100)
values = 2.5 * x + 10.0
feature = create_dataproxy(values)
result = fast_ts_slope(feature, window=20)
# Slope should be approximately 2.5
df = result.df
slopes = df.filter(pl.col("data").is_not_null())["data"].to_numpy()
assert np.abs(slopes[50] - 2.5) < 0.1 # Middle of window
class TestFastTsRsquare:
"""Test fast_ts_rsquare equivalence."""
def test_random_data(self):
"""Exact match on random data."""
np.random.seed(42)
values = np.random.randn(100) * 10 + 50
feature = create_dataproxy(values)
original = ts_function.ts_rsquare(feature, window=10)
fast = fast_ts_rsquare(feature, window=10)
assert_dataproxy_equal(original, fast, rtol=1e-7)
def test_linear_trend(self):
"""Perfect linear trend should have R² = 1."""
x = np.arange(100)
values = 2.5 * x + 10.0
feature = create_dataproxy(values)
result = fast_ts_rsquare(feature, window=20)
df = result.df
r2_values = df.filter(pl.col("data").is_not_null())["data"].to_numpy()
assert r2_values[50] > 0.99 # Should be near 1.0
class TestFastTsResi:
"""Test fast_ts_resi equivalence."""
def test_random_data(self):
"""Exact match on random data."""
np.random.seed(42)
values = np.random.randn(100) * 10 + 50
feature = create_dataproxy(values)
original = ts_function.ts_resi(feature, window=10)
fast = fast_ts_resi(feature, window=10)
assert_dataproxy_equal(original, fast, rtol=1e-7)
class TestFastTsQuantile:
"""Test fast_ts_quantile equivalence."""
def test_random_data(self):
"""Exact match on random data."""
np.random.seed(42)
values = np.random.randn(100) * 10 + 50
feature = create_dataproxy(values)
original = ts_function.ts_quantile(feature, window=10, quantile=0.5)
fast = fast_ts_quantile(feature, window=10, quantile=0.5)
assert_dataproxy_equal(original, fast, rtol=1e-9)
def test_multiple_quantiles(self):
"""Test various quantile values."""
np.random.seed(42)
values = np.random.randn(100) * 10 + 50
feature = create_dataproxy(values)
for q in [0.25, 0.5, 0.75, 0.9]:
original = ts_function.ts_quantile(feature, window=10, quantile=q)
fast = fast_ts_quantile(feature, window=10, quantile=q)
assert_dataproxy_equal(original, fast, rtol=1e-9)
class TestRegistrationAndIntegration:
"""Test registration and vnpy integration."""
def test_registration_idempotent(self):
"""Registration should be idempotent."""
overrides1 = register_fast_ops()
overrides2 = register_fast_ops()
assert overrides1 == overrides2
assert len(overrides1) == 8
def test_expression_override(self):
"""Test that registered functions override vnpy defaults."""
from vnpy.alpha.dataset.utility import EXPRESSION_FUNCTIONS, calculate_by_expression
# Register fast ops
register_fast_ops()
# Check that fast ops are registered
assert "ts_rank" in EXPRESSION_FUNCTIONS
assert "ts_corr" in EXPRESSION_FUNCTIONS
# Test through calculate_by_expression
np.random.seed(42)
values = np.random.randn(20) * 10 + 50
feature = create_dataproxy(values)
# Test ts_rank through expression (pass DataFrame, not DataProxy)
result = calculate_by_expression(feature.df, "ts_rank(data, 5)")
# Should not crash and should return valid DataFrame result
assert result is not None
# calculate_by_expression returns DataFrame, not DataProxy
assert hasattr(result, 'columns') # It's a DataFrame
df = result # result is already a DataFrame
assert (df["data"] >= 0).all()
assert (df["data"] <= 1).all()
def test_fast_ops_registered_count(self):
"""Verify all expected operators are registered."""
overrides = register_fast_ops()
expected = {
"ts_rank", "ts_corr", "ts_cov",
"ts_decay_linear", "ts_slope", "ts_rsquare", "ts_resi", "ts_quantile"
}
assert set(overrides) == expected
class TestEdgeCases:
"""Test edge cases and boundary conditions."""
def test_short_series(self):
"""Series shorter than window should not crash."""
values = np.array([1.0, 2.0, 3.0])
feature = create_dataproxy(values)
# Should not crash
for fast_func in [fast_ts_rank, fast_ts_decay_linear, fast_ts_slope]:
result = fast_func(feature, window=10)
assert result is not None
def test_all_nan(self):
"""All NaN values should be handled."""
values = np.array([np.nan] * 20)
feature = create_dataproxy(values)
for fast_func in [fast_ts_rank, fast_ts_decay_linear, fast_ts_slope]:
result = fast_func(feature, window=5)
assert result is not None
def test_constant_values(self):
"""Constant values should be handled."""
values = np.array([5.0] * 20)
feature = create_dataproxy(values)
for fast_func in [fast_ts_rank, fast_ts_decay_linear, fast_ts_slope]:
result = fast_func(feature, window=5)
assert result is not None
def test_single_value(self):
"""Single value should not crash."""
values = np.array([5.0])
feature = create_dataproxy(values)
for fast_func in [fast_ts_rank, fast_ts_decay_linear, fast_ts_slope]:
result = fast_func(feature, window=5)
assert result is not None
def test_shift_unroll_speed_guard(self):
"""Performance guard to prevent regression to rolling_map."""
import time
n = 200_000
df = pl.DataFrame({
"vt_symbol": ["A"] * n,
"datetime": list(range(n)),
"data": [((i * 37) % 997) / 997 for i in range(n)]
})
feature = DataProxy(df)
t0 = time.time()
fast_ts_rank(feature, 20)
elapsed = time.time() - t0
# Should complete in under 10 seconds for 200k rows
assert elapsed < 10, f"Performance regression: {elapsed:.2f}s > 10s"
if __name__ == "__main__":
pytest.main([__file__, "-v"])