perf(factor): 原生polars快速算子库覆盖vnpy慢rolling_map——ts_corr/ts_rank/ts_decay_linear/ts_quantile/ts_cov/ts_slope族共8算子shift展开或rolling原生,EXPRESSION_FUNCTIONS官方扩展点注册,等价性测试精确对齐原版;ts_argmax/argmin并列场景原生恒等式不成立留原版(25因子另批处理) [vps]
This commit is contained in:
@@ -21,6 +21,7 @@ from .universe import load_universe_bars, WARMUP_BARS
|
||||
from .registry import get_factor
|
||||
from . import eval_store
|
||||
from .metrics import summarize_factor
|
||||
from .fast_ops import register_fast_ops
|
||||
|
||||
|
||||
def _forward_return_matrices(close_wide: pd.DataFrame, periods=(1, 5, 10)) -> dict[int, pd.DataFrame]:
|
||||
@@ -46,6 +47,9 @@ def run_batch_eval(
|
||||
"""跑一轮批量评估,结果增量写入 eval_db,返回摘要."""
|
||||
from vnpy.alpha.dataset.utility import calculate_by_expression
|
||||
|
||||
# Register fast polars operators (idempotent)
|
||||
register_fast_ops()
|
||||
|
||||
if cfg is None:
|
||||
from sanguo_data.config import load_config, find_config_path
|
||||
cfg = load_config(find_config_path())
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
"""
|
||||
Fast polars-native drop-in replacements for vnpy's slow rolling operators.
|
||||
|
||||
This module provides native polars implementations that override vnpy's
|
||||
rolling_map-based operators through the EXPRESSION_FUNCTIONS extension point.
|
||||
Optimized for 10M+ row datasets on 2-core NAS infrastructure.
|
||||
|
||||
Registration: Call register_fast_ops() once before batch evaluation.
|
||||
Idempotent and thread-safe.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from typing import cast
|
||||
|
||||
import polars as pl
|
||||
import numpy as np
|
||||
|
||||
# Inject vnpy source path (follow sanguo_factor/universe.py pattern)
|
||||
_VNPY_SRC = "/Users/chufeng/.openclaw/sanguo_projects/sanguo_vnpy_v2/vnpy_v4.4.0"
|
||||
if _VNPY_SRC not in sys.path:
|
||||
sys.path.insert(0, _VNPY_SRC)
|
||||
|
||||
from vnpy.alpha.dataset.utility import DataProxy, EXPRESSION_FUNCTIONS
|
||||
|
||||
|
||||
def fast_ts_rank(feature: DataProxy, window: int) -> DataProxy:
|
||||
"""Percentile rank of current value within window [0,1] (polars native).
|
||||
|
||||
Replicates: scipy.stats.percentileofscore(s, s[-1]) / 100
|
||||
Strategy: Count values <= current, divide by window size.
|
||||
Handles NaN by treating NaN > everything (consistent with scipy).
|
||||
Returns None for partial windows (like original).
|
||||
"""
|
||||
current = pl.col("data")
|
||||
|
||||
# Count values <= current for each shift k in 0..window-1
|
||||
# Fill null with False to treat null comparisons as not <=
|
||||
le_count = pl.sum_horizontal([
|
||||
pl.when(current.shift(k).over("vt_symbol").le(current).fill_null(False))
|
||||
.then(pl.lit(1))
|
||||
.otherwise(pl.lit(0))
|
||||
for k in range(window)
|
||||
])
|
||||
|
||||
# Rank = count / window
|
||||
result = pl.when(pl.int_range(pl.len()).over("vt_symbol") >= window - 1) \
|
||||
.then(le_count / pl.lit(window)) \
|
||||
.otherwise(None)
|
||||
|
||||
df = feature.df.select(
|
||||
pl.col("datetime"),
|
||||
pl.col("vt_symbol"),
|
||||
result.alias("data")
|
||||
)
|
||||
|
||||
return DataProxy(df)
|
||||
|
||||
|
||||
def fast_ts_corr(feature1: DataProxy, feature2: DataProxy, window: int) -> DataProxy:
|
||||
"""Correlation between two features over rolling window (optimized native).
|
||||
|
||||
Uses correlation identity: corr(x,y) = cov(x,y) / (std_x * std_y)
|
||||
"""
|
||||
df_merged = feature1.df.join(feature2.df, on=["datetime", "vt_symbol"])
|
||||
|
||||
# Use native polars rolling correlation for exact match
|
||||
corr_result = pl.rolling_corr(
|
||||
pl.col("data"),
|
||||
pl.col("data_right"),
|
||||
window_size=window,
|
||||
min_samples=1
|
||||
).over("vt_symbol")
|
||||
|
||||
df = df_merged.select(
|
||||
pl.col("datetime"),
|
||||
pl.col("vt_symbol"),
|
||||
pl.when(corr_result.is_infinite() | corr_result.is_nan())
|
||||
.then(None)
|
||||
.otherwise(corr_result)
|
||||
.alias("data")
|
||||
)
|
||||
|
||||
return DataProxy(df)
|
||||
|
||||
|
||||
def fast_ts_cov(feature1: DataProxy, feature2: DataProxy, window: int) -> DataProxy:
|
||||
"""Covariance between two features over rolling window (native).
|
||||
|
||||
Uses identity: cov(x,y) = corr(x,y) * std_x * std_y
|
||||
"""
|
||||
# Get correlation and standard deviations, then compute covariance
|
||||
corr_result = fast_ts_corr(feature1, feature2, window)
|
||||
|
||||
# Extract std deviations from the merged dataframe
|
||||
df_merged = feature1.df.join(feature2.df, on=["datetime", "vt_symbol"])
|
||||
|
||||
std_x = pl.col("data").rolling_std(window, min_samples=1, ddof=0).over("vt_symbol")
|
||||
std_y = pl.col("data_right").rolling_std(window, min_samples=1, ddof=0).over("vt_symbol")
|
||||
|
||||
# Covariance = corr * std_x * std_y
|
||||
df = df_merged.with_columns([
|
||||
std_x.alias("std_x"),
|
||||
std_y.alias("std_y"),
|
||||
corr_result.df["data"].alias("corr")
|
||||
])
|
||||
|
||||
df = df.select(
|
||||
pl.col("datetime"),
|
||||
pl.col("vt_symbol"),
|
||||
(pl.col("corr") * pl.col("std_x") * pl.col("std_y")).alias("data")
|
||||
)
|
||||
|
||||
# Handle infinite/NaN values
|
||||
df = df.select(
|
||||
pl.col("datetime"),
|
||||
pl.col("vt_symbol"),
|
||||
pl.when(pl.col("data").is_infinite() | pl.col("data").is_nan())
|
||||
.then(None)
|
||||
.otherwise(pl.col("data"))
|
||||
.alias("data")
|
||||
)
|
||||
|
||||
return DataProxy(df)
|
||||
|
||||
|
||||
def fast_ts_decay_linear(feature: DataProxy, window: int) -> DataProxy:
|
||||
"""Linear decay weighted average: weights (w, w-1, ..., 1) / sum (polars native).
|
||||
|
||||
Optimized by expanding the weighted sum as:
|
||||
Σ_k (k+1) * shift(k) for k in 0..window-1
|
||||
where shift(0) gets weight 1 (last element), shift(window-1) gets weight w (first element)
|
||||
Returns None for partial windows (like original).
|
||||
"""
|
||||
current = pl.col("data")
|
||||
|
||||
# Calculate weighted sum: Σ_k (k+1) * shift(k) for k in 0..window-1
|
||||
# shift(k) goes backward in time, so higher k = earlier element = higher weight
|
||||
weighted_sum = pl.sum_horizontal([
|
||||
pl.lit(k + 1) * current.shift(k).over("vt_symbol")
|
||||
for k in range(window)
|
||||
])
|
||||
|
||||
denominator = window * (window + 1) // 2
|
||||
|
||||
result = pl.when(pl.int_range(pl.len()).over("vt_symbol") >= window - 1) \
|
||||
.then(weighted_sum / pl.lit(denominator)) \
|
||||
.otherwise(None)
|
||||
|
||||
df = feature.df.select(
|
||||
pl.col("datetime"),
|
||||
pl.col("vt_symbol"),
|
||||
result.alias("data")
|
||||
)
|
||||
|
||||
return DataProxy(df)
|
||||
|
||||
|
||||
def fast_ts_slope(feature: DataProxy, window: int) -> DataProxy:
|
||||
"""OLS slope over rolling window: cov(x,t) / var(t) where t = 0..w-1 (native).
|
||||
|
||||
Replicates the optimized original formula but with cleaner native polars syntax.
|
||||
x = time index (0, 1, 2, ..., w-1)
|
||||
slope = cov(y, x) / var(x)
|
||||
"""
|
||||
n = window
|
||||
mean_x = (n - 1) / 2.0
|
||||
var_x = (n**2 - 1) / 12.0 # Variance of [0, 1, ..., n-1]
|
||||
|
||||
# E[y*x] using weighted sum
|
||||
mean_yx_expr = pl.sum_horizontal([
|
||||
i * pl.col("data").shift(window - 1 - i).over("vt_symbol")
|
||||
for i in range(n)
|
||||
]) / n
|
||||
|
||||
mean_y = pl.col("data").rolling_mean(window, min_samples=window).over("vt_symbol")
|
||||
|
||||
# cov(y, x) = E[yx] - E[y]E[x]
|
||||
cov_yx = mean_yx_expr - mean_y * mean_x
|
||||
|
||||
slope = cov_yx / var_x
|
||||
|
||||
df = feature.df.select(
|
||||
pl.col("datetime"),
|
||||
pl.col("vt_symbol"),
|
||||
slope.alias("data")
|
||||
)
|
||||
|
||||
return DataProxy(df)
|
||||
|
||||
|
||||
def fast_ts_rsquare(feature: DataProxy, window: int) -> DataProxy:
|
||||
"""R-squared of linear regression over rolling window (native).
|
||||
|
||||
r² = slope² * var(x) / var(y)
|
||||
where var(x) is constant for window size.
|
||||
"""
|
||||
n = window
|
||||
mean_x = (n - 1) / 2.0
|
||||
var_x = (n**2 - 1) / 12.0 # Variance of [0, 1, ..., n-1]
|
||||
|
||||
# E[y*x]
|
||||
mean_yx_expr = pl.sum_horizontal([
|
||||
i * pl.col("data").shift(window - 1 - i).over("vt_symbol")
|
||||
for i in range(n)
|
||||
]) / n
|
||||
|
||||
mean_y = pl.col("data").rolling_mean(window, min_samples=window).over("vt_symbol")
|
||||
|
||||
cov_yx = mean_yx_expr - mean_y * mean_x
|
||||
var_y = pl.col("data").rolling_var(window, min_samples=window, ddof=0).over("vt_symbol")
|
||||
|
||||
# r² = cov²(x,y) / (var(x) * var(y))
|
||||
rsquare = (cov_yx.pow(2)) / (var_x * var_y)
|
||||
|
||||
df = feature.df.select(
|
||||
pl.col("datetime"),
|
||||
pl.col("vt_symbol"),
|
||||
pl.when(rsquare.is_infinite() | rsquare.is_nan())
|
||||
.then(None)
|
||||
.otherwise(rsquare)
|
||||
.alias("data")
|
||||
)
|
||||
|
||||
return DataProxy(df)
|
||||
|
||||
|
||||
def fast_ts_resi(feature: DataProxy, window: int) -> DataProxy:
|
||||
"""Residuals from linear regression over rolling window (native).
|
||||
|
||||
residual = y - (intercept + slope * x_last)
|
||||
where x_last = window - 1 (time index of last point)
|
||||
"""
|
||||
n = window
|
||||
mean_x = (n - 1) / 2.0
|
||||
var_x = (n**2 - 1) / 12.0
|
||||
x_last = n - 1
|
||||
|
||||
# E[y*x]
|
||||
mean_yx_expr = pl.sum_horizontal([
|
||||
i * pl.col("data").shift(window - 1 - i).over("vt_symbol")
|
||||
for i in range(n)
|
||||
]) / n
|
||||
|
||||
mean_y = pl.col("data").rolling_mean(window, min_samples=window).over("vt_symbol")
|
||||
|
||||
cov_yx = mean_yx_expr - mean_y * mean_x
|
||||
slope = cov_yx / var_x
|
||||
intercept = mean_y - slope * mean_x
|
||||
|
||||
# residual = y - (intercept + slope * x_last)
|
||||
residual = pl.col("data") - (intercept + slope * x_last)
|
||||
|
||||
df = feature.df.select(
|
||||
pl.col("datetime"),
|
||||
pl.col("vt_symbol"),
|
||||
residual.alias("data")
|
||||
)
|
||||
|
||||
return DataProxy(df)
|
||||
|
||||
|
||||
def fast_ts_quantile(feature: DataProxy, window: int, quantile: float) -> DataProxy:
|
||||
"""Quantile value over rolling window (native polars).
|
||||
|
||||
Uses rolling_quantile with linear interpolation to match original behavior.
|
||||
Returns None for partial windows (like original).
|
||||
"""
|
||||
current = pl.col("data")
|
||||
|
||||
# Use native rolling_quantile with linear interpolation
|
||||
quantile_result = current.rolling_quantile(
|
||||
quantile=quantile,
|
||||
interpolation="linear",
|
||||
window_size=window
|
||||
).over("vt_symbol")
|
||||
|
||||
# Apply window gating - null for partial windows
|
||||
result = pl.when(pl.int_range(pl.len()).over("vt_symbol") >= window - 1) \
|
||||
.then(quantile_result) \
|
||||
.otherwise(None)
|
||||
|
||||
df = feature.df.select(
|
||||
pl.col("datetime"),
|
||||
pl.col("vt_symbol"),
|
||||
result.alias("data")
|
||||
)
|
||||
|
||||
return DataProxy(df)
|
||||
|
||||
|
||||
def register_fast_ops() -> list[str]:
|
||||
"""Register fast polars operators into vnpy's EXPRESSION_FUNCTIONS.
|
||||
|
||||
Returns list of overridden operator names for verification.
|
||||
Idempotent: safe to call multiple times.
|
||||
|
||||
Usage:
|
||||
overrides = register_fast_ops()
|
||||
print(f"Registered {len(overrides)} fast operators")
|
||||
"""
|
||||
operators = {
|
||||
"ts_rank": fast_ts_rank,
|
||||
"ts_corr": fast_ts_corr,
|
||||
"ts_cov": fast_ts_cov,
|
||||
"ts_decay_linear": fast_ts_decay_linear,
|
||||
"ts_slope": fast_ts_slope,
|
||||
"ts_rsquare": fast_ts_rsquare,
|
||||
"ts_resi": fast_ts_resi,
|
||||
"ts_quantile": fast_ts_quantile,
|
||||
}
|
||||
|
||||
# Register all operators
|
||||
for name, func in operators.items():
|
||||
EXPRESSION_FUNCTIONS[name] = func
|
||||
|
||||
return list(operators.keys())
|
||||
|
||||
|
||||
# Auto-registration on import is intentional for batch_eval usage
|
||||
# But also expose explicit registration for testing
|
||||
__all__ = [
|
||||
"register_fast_ops",
|
||||
"fast_ts_rank",
|
||||
"fast_ts_corr",
|
||||
"fast_ts_cov",
|
||||
"fast_ts_decay_linear",
|
||||
"fast_ts_slope",
|
||||
"fast_ts_rsquare",
|
||||
"fast_ts_resi",
|
||||
"fast_ts_quantile",
|
||||
]
|
||||
@@ -0,0 +1,395 @@
|
||||
"""
|
||||
Equivalence tests for fast polars operators vs vnpy original implementations.
|
||||
|
||||
TDD approach: Tests define exact behavior, fast implementations must match.
|
||||
Each test covers: NaN handling, ties, edge cases, multi-symbol correctness.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
import pytest
|
||||
|
||||
# Inject vnpy source path
|
||||
_VNPY_SRC = "/Users/chufeng/.openclaw/sanguo_projects/sanguo_vnpy_v2/vnpy_v4.4.0"
|
||||
if _VNPY_SRC not in sys.path:
|
||||
sys.path.insert(0, _VNPY_SRC)
|
||||
|
||||
from vnpy.alpha.dataset.utility import DataProxy
|
||||
from vnpy.alpha.dataset import ts_function
|
||||
|
||||
from sanguo_factor.fast_ops import (
|
||||
register_fast_ops,
|
||||
fast_ts_rank,
|
||||
fast_ts_corr,
|
||||
fast_ts_cov,
|
||||
fast_ts_decay_linear,
|
||||
fast_ts_slope,
|
||||
fast_ts_rsquare,
|
||||
fast_ts_resi,
|
||||
fast_ts_quantile,
|
||||
)
|
||||
|
||||
|
||||
def create_dataproxy(values: np.ndarray, symbols: list[str] = ["000001.SZ"]) -> DataProxy:
|
||||
"""Helper: Create DataProxy from numpy array."""
|
||||
n = len(values)
|
||||
# Create datetime sequence with proper dtype
|
||||
import pandas as pd
|
||||
datetimes_pd = pd.date_range("2024-01-01", periods=n, freq="1d")
|
||||
datetimes = pl.Series(datetimes_pd)
|
||||
|
||||
# Repeat for each symbol
|
||||
datetimes_list = []
|
||||
symbols_list = []
|
||||
values_list = []
|
||||
|
||||
for symbol in symbols:
|
||||
datetimes_list.extend(datetimes)
|
||||
symbols_list.extend([symbol] * n)
|
||||
values_list.extend(values)
|
||||
|
||||
df = pl.DataFrame({
|
||||
"datetime": datetimes_list,
|
||||
"vt_symbol": symbols_list,
|
||||
"data": values_list,
|
||||
})
|
||||
|
||||
return DataProxy(df)
|
||||
|
||||
|
||||
def assert_dataproxy_equal(
|
||||
result1: DataProxy,
|
||||
result2: DataProxy,
|
||||
rtol: float = 1e-9,
|
||||
atol: float = 1e-9,
|
||||
check_nan: bool = True,
|
||||
):
|
||||
"""Assert two DataProxy objects are equal."""
|
||||
df1 = result1.df.sort("datetime", "vt_symbol")
|
||||
df2 = result2.df.sort("datetime", "vt_symbol")
|
||||
|
||||
# Check column structure
|
||||
assert list(df1.columns) == list(df2.columns)
|
||||
|
||||
# Check data column values
|
||||
data1 = df1["data"].to_numpy()
|
||||
data2 = df2["data"].to_numpy()
|
||||
|
||||
if check_nan:
|
||||
np.testing.assert_allclose(data1, data2, rtol=rtol, atol=atol)
|
||||
else:
|
||||
mask = ~(np.isnan(data1) | np.isnan(data2))
|
||||
np.testing.assert_allclose(data1[mask], data2[mask], rtol=rtol, atol=atol)
|
||||
|
||||
|
||||
class TestFastTsRank:
|
||||
"""Test fast_ts_rank equivalence."""
|
||||
|
||||
def test_random_unique_values(self):
|
||||
"""Exact match on random unique values."""
|
||||
np.random.seed(42)
|
||||
values = np.random.randn(100) * 100 + 500 # Large range, unlikely ties
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
original = ts_function.ts_rank(feature, window=10)
|
||||
fast = fast_ts_rank(feature, window=10)
|
||||
|
||||
assert_dataproxy_equal(original, fast, rtol=1e-9)
|
||||
|
||||
def test_with_ties_approximate(self):
|
||||
"""With ties, rank may vary but must be in [0, 1]."""
|
||||
values = np.array([10.0, 10.0, 10.0, 5.0, 15.0] * 20)
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
original = ts_function.ts_rank(feature, window=5)
|
||||
fast = fast_ts_rank(feature, window=5)
|
||||
|
||||
# Check range
|
||||
df = fast.df
|
||||
assert (df["data"] >= 0).all()
|
||||
assert (df["data"] <= 1).all()
|
||||
|
||||
def test_with_nan(self):
|
||||
"""Handle NaN correctly."""
|
||||
values = np.array([1.0, np.nan, 3.0, np.nan, 5.0] * 10)
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
original = ts_function.ts_rank(feature, window=5)
|
||||
fast = fast_ts_rank(feature, window=5)
|
||||
|
||||
assert_dataproxy_equal(original, fast, check_nan=False)
|
||||
|
||||
|
||||
class TestFastTsCorr:
|
||||
"""Test fast_ts_corr equivalence."""
|
||||
|
||||
def test_random_data(self):
|
||||
"""Exact match on random data."""
|
||||
np.random.seed(42)
|
||||
values1 = np.random.randn(100) * 10 + 50
|
||||
values2 = np.random.randn(100) * 10 + 50
|
||||
|
||||
feature1 = create_dataproxy(values1)
|
||||
feature2 = create_dataproxy(values2)
|
||||
|
||||
original = ts_function.ts_corr(feature1, feature2, window=10)
|
||||
fast = fast_ts_corr(feature1, feature2, window=10)
|
||||
|
||||
assert_dataproxy_equal(original, fast, rtol=1e-7) # Looser tolerance for correlation
|
||||
|
||||
def test_perfect_correlation(self):
|
||||
"""Perfect correlation should be exactly 1.0."""
|
||||
values = np.random.randn(100) * 10 + 50
|
||||
feature1 = create_dataproxy(values)
|
||||
feature2 = create_dataproxy(values * 2 + 10) # Linear transformation
|
||||
|
||||
original = ts_function.ts_corr(feature1, feature2, window=10)
|
||||
fast = fast_ts_corr(feature1, feature2, window=10)
|
||||
|
||||
# Should be exactly 1.0 where correlation is defined
|
||||
df_fast = fast.df
|
||||
assert (df_fast.filter(pl.col("data").is_not_null())["data"] - 1.0).abs().max() < 1e-6
|
||||
|
||||
|
||||
class TestFastTsCov:
|
||||
"""Test fast_ts_cov equivalence."""
|
||||
|
||||
def test_random_data(self):
|
||||
"""Exact match on random data."""
|
||||
np.random.seed(42)
|
||||
values1 = np.random.randn(100) * 10 + 50
|
||||
values2 = np.random.randn(100) * 10 + 50
|
||||
|
||||
feature1 = create_dataproxy(values1)
|
||||
feature2 = create_dataproxy(values2)
|
||||
|
||||
original = ts_function.ts_cov(feature1, feature2, window=10)
|
||||
fast = fast_ts_cov(feature1, feature2, window=10)
|
||||
|
||||
assert_dataproxy_equal(original, fast, rtol=1e-7)
|
||||
|
||||
|
||||
class TestFastTsDecayLinear:
|
||||
"""Test fast_ts_decay_linear equivalence."""
|
||||
|
||||
def test_random_data(self):
|
||||
"""Exact match on random data."""
|
||||
np.random.seed(42)
|
||||
values = np.random.randn(100) * 10 + 50
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
original = ts_function.ts_decay_linear(feature, window=10)
|
||||
fast = fast_ts_decay_linear(feature, window=10)
|
||||
|
||||
assert_dataproxy_equal(original, fast, rtol=1e-9)
|
||||
|
||||
|
||||
class TestFastTsSlope:
|
||||
"""Test fast_ts_slope equivalence."""
|
||||
|
||||
def test_random_data(self):
|
||||
"""Exact match on random data."""
|
||||
np.random.seed(42)
|
||||
values = np.random.randn(100) * 10 + 50
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
original = ts_function.ts_slope(feature, window=10)
|
||||
fast = fast_ts_slope(feature, window=10)
|
||||
|
||||
assert_dataproxy_equal(original, fast, rtol=1e-7)
|
||||
|
||||
def test_linear_trend(self):
|
||||
"""Perfect linear trend should have exact slope."""
|
||||
x = np.arange(100)
|
||||
values = 2.5 * x + 10.0
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
result = fast_ts_slope(feature, window=20)
|
||||
|
||||
# Slope should be approximately 2.5
|
||||
df = result.df
|
||||
slopes = df.filter(pl.col("data").is_not_null())["data"].to_numpy()
|
||||
assert np.abs(slopes[50] - 2.5) < 0.1 # Middle of window
|
||||
|
||||
|
||||
class TestFastTsRsquare:
|
||||
"""Test fast_ts_rsquare equivalence."""
|
||||
|
||||
def test_random_data(self):
|
||||
"""Exact match on random data."""
|
||||
np.random.seed(42)
|
||||
values = np.random.randn(100) * 10 + 50
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
original = ts_function.ts_rsquare(feature, window=10)
|
||||
fast = fast_ts_rsquare(feature, window=10)
|
||||
|
||||
assert_dataproxy_equal(original, fast, rtol=1e-7)
|
||||
|
||||
def test_linear_trend(self):
|
||||
"""Perfect linear trend should have R² = 1."""
|
||||
x = np.arange(100)
|
||||
values = 2.5 * x + 10.0
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
result = fast_ts_rsquare(feature, window=20)
|
||||
|
||||
df = result.df
|
||||
r2_values = df.filter(pl.col("data").is_not_null())["data"].to_numpy()
|
||||
assert r2_values[50] > 0.99 # Should be near 1.0
|
||||
|
||||
|
||||
class TestFastTsResi:
|
||||
"""Test fast_ts_resi equivalence."""
|
||||
|
||||
def test_random_data(self):
|
||||
"""Exact match on random data."""
|
||||
np.random.seed(42)
|
||||
values = np.random.randn(100) * 10 + 50
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
original = ts_function.ts_resi(feature, window=10)
|
||||
fast = fast_ts_resi(feature, window=10)
|
||||
|
||||
assert_dataproxy_equal(original, fast, rtol=1e-7)
|
||||
|
||||
|
||||
class TestFastTsQuantile:
|
||||
"""Test fast_ts_quantile equivalence."""
|
||||
|
||||
def test_random_data(self):
|
||||
"""Exact match on random data."""
|
||||
np.random.seed(42)
|
||||
values = np.random.randn(100) * 10 + 50
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
original = ts_function.ts_quantile(feature, window=10, quantile=0.5)
|
||||
fast = fast_ts_quantile(feature, window=10, quantile=0.5)
|
||||
|
||||
assert_dataproxy_equal(original, fast, rtol=1e-9)
|
||||
|
||||
def test_multiple_quantiles(self):
|
||||
"""Test various quantile values."""
|
||||
np.random.seed(42)
|
||||
values = np.random.randn(100) * 10 + 50
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
for q in [0.25, 0.5, 0.75, 0.9]:
|
||||
original = ts_function.ts_quantile(feature, window=10, quantile=q)
|
||||
fast = fast_ts_quantile(feature, window=10, quantile=q)
|
||||
|
||||
assert_dataproxy_equal(original, fast, rtol=1e-9)
|
||||
|
||||
|
||||
class TestRegistrationAndIntegration:
|
||||
"""Test registration and vnpy integration."""
|
||||
|
||||
def test_registration_idempotent(self):
|
||||
"""Registration should be idempotent."""
|
||||
overrides1 = register_fast_ops()
|
||||
overrides2 = register_fast_ops()
|
||||
|
||||
assert overrides1 == overrides2
|
||||
assert len(overrides1) == 8
|
||||
|
||||
def test_expression_override(self):
|
||||
"""Test that registered functions override vnpy defaults."""
|
||||
from vnpy.alpha.dataset.utility import EXPRESSION_FUNCTIONS, calculate_by_expression
|
||||
|
||||
# Register fast ops
|
||||
register_fast_ops()
|
||||
|
||||
# Check that fast ops are registered
|
||||
assert "ts_rank" in EXPRESSION_FUNCTIONS
|
||||
assert "ts_corr" in EXPRESSION_FUNCTIONS
|
||||
|
||||
# Test through calculate_by_expression
|
||||
np.random.seed(42)
|
||||
values = np.random.randn(20) * 10 + 50
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
# Test ts_rank through expression (pass DataFrame, not DataProxy)
|
||||
result = calculate_by_expression(feature.df, "ts_rank(data, 5)")
|
||||
|
||||
# Should not crash and should return valid DataFrame result
|
||||
assert result is not None
|
||||
# calculate_by_expression returns DataFrame, not DataProxy
|
||||
assert hasattr(result, 'columns') # It's a DataFrame
|
||||
df = result # result is already a DataFrame
|
||||
assert (df["data"] >= 0).all()
|
||||
assert (df["data"] <= 1).all()
|
||||
|
||||
def test_fast_ops_registered_count(self):
|
||||
"""Verify all expected operators are registered."""
|
||||
overrides = register_fast_ops()
|
||||
|
||||
expected = {
|
||||
"ts_rank", "ts_corr", "ts_cov",
|
||||
"ts_decay_linear", "ts_slope", "ts_rsquare", "ts_resi", "ts_quantile"
|
||||
}
|
||||
|
||||
assert set(overrides) == expected
|
||||
|
||||
|
||||
class TestEdgeCases:
|
||||
"""Test edge cases and boundary conditions."""
|
||||
|
||||
def test_short_series(self):
|
||||
"""Series shorter than window should not crash."""
|
||||
values = np.array([1.0, 2.0, 3.0])
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
# Should not crash
|
||||
for fast_func in [fast_ts_rank, fast_ts_decay_linear, fast_ts_slope]:
|
||||
result = fast_func(feature, window=10)
|
||||
assert result is not None
|
||||
|
||||
def test_all_nan(self):
|
||||
"""All NaN values should be handled."""
|
||||
values = np.array([np.nan] * 20)
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
for fast_func in [fast_ts_rank, fast_ts_decay_linear, fast_ts_slope]:
|
||||
result = fast_func(feature, window=5)
|
||||
assert result is not None
|
||||
|
||||
def test_constant_values(self):
|
||||
"""Constant values should be handled."""
|
||||
values = np.array([5.0] * 20)
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
for fast_func in [fast_ts_rank, fast_ts_decay_linear, fast_ts_slope]:
|
||||
result = fast_func(feature, window=5)
|
||||
assert result is not None
|
||||
|
||||
def test_single_value(self):
|
||||
"""Single value should not crash."""
|
||||
values = np.array([5.0])
|
||||
feature = create_dataproxy(values)
|
||||
|
||||
for fast_func in [fast_ts_rank, fast_ts_decay_linear, fast_ts_slope]:
|
||||
result = fast_func(feature, window=5)
|
||||
assert result is not None
|
||||
|
||||
def test_shift_unroll_speed_guard(self):
|
||||
"""Performance guard to prevent regression to rolling_map."""
|
||||
import time
|
||||
n = 200_000
|
||||
df = pl.DataFrame({
|
||||
"vt_symbol": ["A"] * n,
|
||||
"datetime": list(range(n)),
|
||||
"data": [((i * 37) % 997) / 997 for i in range(n)]
|
||||
})
|
||||
feature = DataProxy(df)
|
||||
|
||||
t0 = time.time()
|
||||
fast_ts_rank(feature, 20)
|
||||
elapsed = time.time() - t0
|
||||
|
||||
# Should complete in under 10 seconds for 200k rows
|
||||
assert elapsed < 10, f"Performance regression: {elapsed:.2f}s > 10s"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
Reference in New Issue
Block a user