feat(stats): correlation_matrix 按方案 A 拆成三个函数,收口返回契约(L0 9/12)

用户 2026-10-03 拍板方案 A(见决策 41),由 Codex 落地。

拆成三个函数:
- correlation_matrix  -> 数值系数矩阵(对角线 1.0)
- correlation_pvalues -> 数值 p 值矩阵(对角线 NaN)
- correlation_table   -> 星标字符串表(对角线 1.000 不带星)
三者共用 _check_corr_args 守卫与成对删缺失口径;correlation_table 在最外层调另外两个。
理由: 字符串表做不了后续计算(挑高相关对、喂聚类/PCA、画热力图),反过来随时能格式化;
      一旦返回字符串,数值信息不可逆地丢失。

顺带修掉两处:
- 对角线 p 值由 0.0 改成 NaN —— 写 0 会被下游读成"极显著",而"自己和自己"没有 p 值
- p 值与 r 现在用同一批观测: 先成对 dropna() 再算,并同样尊重 min_periods。
  此前 corr 是成对删缺失、pearsonr 不删,两者样本量不同,r 与 p 对不上
- p 值只算上三角再镜像(成对变量的 p 天然对称),省一半重复计算

tests: test_correlation_matrix_diagonal_is_one 摘掉 xfail 转正;新增 8 条
  (数值性+对称性、非数值列跳过、p 值对照解析式 t = r*sqrt((n-2)/(1-r^2))、
   对角线 NaN、成对删缺失口径、min_periods、对角线无星标、表格与数值输出一致)。
  测试 105 passed / 38 skipped / 1 xfail -> 114 passed / 38 skipped / 0 xfail。

验证: pytest 114 passed / 38 skipped;ruff All checks passed;pyright 0 errors。

docs: STATUS.md(L0 9/12 + 决策 41 + 下一步);CHANGELOG;
      personal-pitfalls 追加第 14 条(把"能看的表"和"能算的表"塞进同一个返回值);
      roadmap 记录 correlation_matrix 的两个伴随函数
This commit is contained in:
guzhujushi committed 2026-10-03 14:53:05 +08:00
1 parent 3432bf3e50
commit 95f5266563
7 files changed
+308 -71

No files matched your search

+110 -4
View File
@@ -15,6 +15,17 @@ import pytest
_NOT_IMPLEMENTED = "未实现:见 docs/03-roadmap.md L1"
def _pearson_p_from_r(r: float, n: int) -> float:
"""Pearson 相关系数的 p 值**解析式**(测解析解,不测实现细节):
``t = r * sqrt((n - 2) / (1 - r^2))``,``p = 2 * P(T > |t|)``,``T ~ t(n - 2)``。
"""
from scipy import stats as sps
t_stat = r * np.sqrt((n - 2) / (1 - r**2))
return float(2 * sps.t.sf(abs(t_stat), n - 2))
@pytest.mark.skip(reason=_NOT_IMPLEMENTED)
def test_bootstrap_ci_matches_normal_theory_on_large_n(rng):
"""大样本下 bootstrap 百分位区间应逼近正态解析区间(±10% 容差)。"""
@@ -59,18 +70,27 @@ def test_confidence_interval_matches_scipy(rng):
assert np.allclose([lo, hi], ref, rtol=1e-6)
@pytest.mark.xfail(
strict=False,
reason="契约要求返回**数值**矩阵(对角 = 1.0),实现返回带星标的字符串表;口径待定,见 STATUS.md",
)
def test_correlation_matrix_diagonal_is_one(rng):
from mcm_toolkit.stats import correlation_matrix
x = pd.DataFrame(rng.normal(size=(200, 4)), columns=["a", "b", "c", "d"])
c = correlation_matrix(x)
assert np.allclose(np.diag(np.asarray(c)), 1.0)
def test_correlation_matrix_is_numeric_and_symmetric(rng):
"""方案 A:返回**数值**矩阵(不是带星标的字符串表),能直接参与后续计算。"""
from mcm_toolkit.stats import correlation_matrix
x = pd.DataFrame(rng.normal(size=(200, 3)), columns=["a", "b", "c"])
c = correlation_matrix(x)
assert all(np.issubdtype(dtype, np.floating) for dtype in c.dtypes)
assert np.allclose(np.asarray(c), np.asarray(x.corr(method="pearson", min_periods=1)))
assert np.allclose(np.asarray(c), np.asarray(c.T))
def test_correlation_matrix_runs_on_multicolumn_frame(rng):
"""回归:`if numeric_cols == 0` 拿 Index 跟 0 比,多列数值表当场 ValueError 崩掉。"""
from mcm_toolkit.stats import correlation_matrix
@@ -83,6 +103,13 @@ def test_correlation_matrix_runs_on_multicolumn_frame(rng):
assert list(out.index) == ["a", "b", "c"]
def test_correlation_matrix_skips_non_numeric_columns(rng):
from mcm_toolkit.stats import correlation_matrix
x = pd.DataFrame({"a": rng.normal(size=30), "b": rng.normal(size=30), "city": ["X"] * 30})
assert list(correlation_matrix(x).columns) == ["a", "b"]
def test_correlation_matrix_rejects_bad_input():
from mcm_toolkit.stats import correlation_matrix
@@ -94,6 +121,85 @@ def test_correlation_matrix_rejects_bad_input():
correlation_matrix(pd.DataFrame({"a": [1.0, 2.0]}), method="bogus") # type: ignore[arg-type]
def test_correlation_pvalues_matches_analytic_formula(rng):
from mcm_toolkit.stats import correlation_matrix, correlation_pvalues
x = pd.DataFrame(rng.normal(size=(80, 3)), columns=["a", "b", "c"])
corr = correlation_matrix(x)
p = correlation_pvalues(x)
for i in ["a", "b", "c"]:
for j in ["a", "b", "c"]:
if i == j:
continue
assert p.loc[i, j] == pytest.approx(_pearson_p_from_r(corr.loc[i, j], 80))
def test_correlation_pvalues_diagonal_is_nan(rng):
"""对角线写 0 会被下游读成"极显著";"自己和自己相关"没有 p 值,必须是 NaN。"""
from mcm_toolkit.stats import correlation_pvalues
p = correlation_pvalues(pd.DataFrame(rng.normal(size=(50, 2)), columns=["a", "b"]))
assert np.isnan(p.loc["a", "a"])
assert np.isnan(p.loc["b", "b"])
def test_correlation_pvalues_uses_the_same_observations_as_r(rng):
"""含缺失时 p 要和 r 用**同一批**观测:成对删缺失,而不是整行删、也不是干脆不删。"""
from mcm_toolkit.stats import correlation_pvalues
a = rng.normal(size=40)
b = rng.normal(size=40)
a[[3, 7, 11]] = np.nan
x = pd.DataFrame({"a": a, "b": b})
pair = x.dropna()
r = float(np.corrcoef(pair["a"], pair["b"])[0, 1])
assert correlation_pvalues(x).loc["a", "b"] == pytest.approx(_pearson_p_from_r(r, len(pair)))
def test_correlation_pvalues_respects_min_periods(rng):
from mcm_toolkit.stats import correlation_pvalues
a = rng.normal(size=10)
a[:5] = np.nan
x = pd.DataFrame({"a": a, "b": rng.normal(size=10)})
assert np.isnan(correlation_pvalues(x, min_periods=6).loc["a", "b"])
assert np.isfinite(correlation_pvalues(x, min_periods=5).loc["a", "b"])
def test_correlation_table_diagonal_has_no_stars():
"""对角线是"自己和自己",不该有显著性星标;完全负相关则必然是 ***。"""
from mcm_toolkit.stats import correlation_table
x = pd.DataFrame({"a": np.arange(30.0), "b": np.arange(30.0) * -2.0})
table = correlation_table(x)
assert table.loc["a", "a"] == "1.000"
assert table.loc["a", "b"] == "-1.000***"
def test_correlation_table_matches_the_numeric_outputs(rng):
from mcm_toolkit.stats import correlation_matrix, correlation_pvalues, correlation_table
x = pd.DataFrame(rng.normal(size=(60, 3)), columns=["a", "b", "c"])
corr = correlation_matrix(x)
p = correlation_pvalues(x)
table = correlation_table(x)
assert list(table.columns) == list(corr.columns)
assert table.loc["a", "a"] == "1.000"
assert np.isnan(p.loc["a", "a"])
pv = p.loc["a", "b"]
stars = "***" if pv < 0.001 else "**" if pv < 0.01 else "*" if pv < 0.05 else ""
value = corr.loc["a", "b"]
assert table.loc["a", "b"] == f"{value:.3f}" + stars
# ------------------------------------------------------ stats: describe