feat(stats): describe 落地,修掉一个静默填 NaN 的 bug(L0 8/12)

src/stats/descriptive.py 由用户本人实现,这里做审查 + 代修。

describe:
- 真 bug:df_num.describe().T 把表转反(行=变量、列=指标),而 docstring 与
  test_describe_reports_expected_keys 定的是"行=指标、列=变量"。转置后
  .loc[指标名] = <按变量索引的 Series> 与列名零交集,pandas 不报错,直接把
  missing / missing_ratio / skewness / kurtosis 四行整行填成 NaN(静默丢数据);
  weights 分支则报 cannot set a row with mismatched columns。去掉 .T 一处修好三件事
- 补 median 行(pandas 只有 50%,测试契约定的是 median)
- 入参放宽到 list / tuple / ndarray(00_quickstart.py 传的就是 ndarray)
- weights 校验补齐:维度 / 长度 / NaN / 负值 / 和为 0(原先和为 0 时裸抛 ZeroDivisionError)
- docstring 按 4.7 补 Parameters / Returns / Raises

zscore: 修 RET505;docstring 补齐;DataFrame 取列处按坑 8 用 cast 收窄
coefficient_of_variation: 均值为 0 改抛 ValueError(取值问题不是类型问题);
  空数组提前报错(原先静默 nan + RuntimeWarning);错别字与文案修正

tests: 去掉 test_stats.py 的模块级 pytestmark —— 它把 6 条测试全部跳过(含已实现的
  describe),正是决策 26 要禁的写法。改为逐条 skip 未实现的 5 条,并新增 20 条用例。

验证:pytest 102 passed / 39 skipped(原 82/40);ruff All checks passed;pyright 0 errors。
examples/01_clean_and_describe.py 填掉 TODO(L0),实测 exit 0。

docs: STATUS.md L0 8/12 + 决策 35/36 + 三处待拍板口径;CHANGELOG 记录;
      personal-pitfalls 追加第 11 条(pandas 对不齐就静默填 NaN)
This commit is contained in:
guzhujushi committed 2026-10-03 13:17:58 +08:00
1 parent 4552aee41c
commit e9624cb3a7
6 files changed
+468 -34

No files matched your search

+176 -1
View File
@@ -1,16 +1,21 @@
"""stats 包测试。
原则(见 docs/02-architecture.md §5):**测已知解析解,不测实现细节**。
未实现的函数**逐条** `skip`——**不要**用模块级 `pytestmark`:那会把同文件里
已经写好的函数一起跳过,绿灯就是假的(决策 26 / 个人易错点 9)。
"""
from __future__ import annotations
import numpy as np
import pandas as pd
import pytest
pytestmark = pytest.mark.skip(reason="未实现:见 docs/03-roadmap.md L1")
_NOT_IMPLEMENTED = "未实现:见 docs/03-roadmap.md L1"
@pytest.mark.skip(reason=_NOT_IMPLEMENTED)
def test_bootstrap_ci_matches_normal_theory_on_large_n(rng):
"""大样本下 bootstrap 百分位区间应逼近正态解析区间(±10% 容差)。"""
from mcm_toolkit.stats import bootstrap_ci
@@ -21,6 +26,7 @@ def test_bootstrap_ci_matches_normal_theory_on_large_n(rng):
assert abs((r.params["high"] - r.params["low"]) - 2 * 1.96 * se) < 0.1 * 2 * 1.96 * se
@pytest.mark.skip(reason=_NOT_IMPLEMENTED)
def test_bootstrap_ci_works_for_arbitrary_statistic(rng):
"""bootstrap 的核心价值:任意统计量都能给区间(这里用中位数)。"""
from mcm_toolkit.stats import bootstrap_ci
@@ -31,6 +37,7 @@ def test_bootstrap_ci_works_for_arbitrary_statistic(rng):
assert r.params["low"] < np.median(x) < r.params["high"]
@pytest.mark.skip(reason=_NOT_IMPLEMENTED)
def test_bootstrap_ci_is_reproducible(rng):
from mcm_toolkit.stats import bootstrap_ci
@@ -40,6 +47,7 @@ def test_bootstrap_ci_is_reproducible(rng):
assert a.params["low"] == b.params["low"]
@pytest.mark.skip(reason=_NOT_IMPLEMENTED)
def test_confidence_interval_matches_scipy(rng):
from scipy import stats
@@ -51,6 +59,7 @@ def test_confidence_interval_matches_scipy(rng):
assert np.allclose([lo, hi], ref, rtol=1e-6)
@pytest.mark.skip(reason=_NOT_IMPLEMENTED)
def test_correlation_matrix_diagonal_is_one(rng):
from mcm_toolkit.stats import correlation_matrix
@@ -59,9 +68,175 @@ def test_correlation_matrix_diagonal_is_one(rng):
assert np.allclose(np.diag(np.asarray(c)), 1.0)
# ------------------------------------------------------ stats: describe
def test_describe_reports_expected_keys(rng):
from mcm_toolkit.stats import describe
table = describe(rng.normal(size=(100, 3)))
for key in ["mean", "std", "min", "max", "median"]:
assert key in table.index
def test_describe_rows_are_metrics_and_columns_are_variables():
"""方向约定:行 = 指标,列 = 变量。多转置一次就反了。"""
from mcm_toolkit.stats import describe
df = pd.DataFrame({"a": [1.0, 2.0, 3.0], "b": [10.0, 20.0, 30.0]})
table = describe(df)
assert list(table.columns) == ["a", "b"]
assert table.loc["mean", "a"] == pytest.approx(2.0)
assert table.loc["max", "b"] == pytest.approx(30.0)
assert table.loc["median", "b"] == pytest.approx(20.0)
def test_describe_missing_and_shape_rows_are_not_nan():
"""曾经的静默 bug:转置后 `.loc[指标名] = <按变量索引的 Series>` 对不上列,
整行会被填成 NaN —— 不报错,但数据全丢。"""
from mcm_toolkit.stats import describe
df = pd.DataFrame({"a": [1.0, np.nan, 3.0, 4.0], "b": [1.0, 2.0, 3.0, 4.0]})
table = describe(df)
assert table.loc["missing", "a"] == pytest.approx(1.0)
assert table.loc["missing", "b"] == pytest.approx(0.0)
assert table.loc["missing_ratio", "a"] == pytest.approx(0.25)
assert table.loc["missing_ratio", "b"] == pytest.approx(0.0)
assert table.loc["skewness", "b"] == pytest.approx(0.0, abs=1e-12)
extra = table.loc[["missing", "missing_ratio", "skewness", "kurtosis"]]
assert not bool(extra.isna().all(axis=None))
def test_describe_skips_non_numeric_columns():
from mcm_toolkit.stats import describe
df = pd.DataFrame({"a": [1.0, 2.0], "city": ["X", "Y"]})
table = describe(df)
assert list(table.columns) == ["a"]
def test_describe_accepts_ndarray_list_and_series(rng):
"""兄弟函数都收 ndarray / list,describe 没理由只收 DataFrame。"""
from mcm_toolkit.stats import describe
assert "mean" in describe(rng.normal(size=(50, 2))).index
assert "mean" in describe([[1.0, 2.0], [3.0, 4.0]]).index
assert "mean" in describe(pd.Series([1.0, 2.0, 3.0])).index
def test_describe_weighted_rows_match_manual_formula():
from mcm_toolkit.stats import describe
df = pd.DataFrame({"a": [1.0, 2.0, 3.0]})
w = np.array([1.0, 1.0, 2.0])
table = describe(df, weights=w)
expect_mean = np.average(df["a"], weights=w)
expect_std = np.sqrt(np.average((df["a"] - expect_mean) ** 2, weights=w))
assert table.loc["weighted_mean", "a"] == pytest.approx(expect_mean)
assert table.loc["weighted_std", "a"] == pytest.approx(expect_std)
@pytest.mark.parametrize(
("weights", "exc"),
[
(np.ones(2), ValueError),
(np.zeros(3), ValueError),
(np.array([1.0, -1.0, 1.0]), ValueError),
(np.array([1.0, np.nan, 1.0]), ValueError),
("abc", TypeError),
],
)
def test_describe_rejects_bad_weights(weights, exc):
from mcm_toolkit.stats import describe
with pytest.raises(exc):
describe(pd.DataFrame({"a": [1.0, 2.0, 3.0]}), weights=weights)
def test_describe_rejects_unsupported_type_and_all_text_frame():
from mcm_toolkit.stats import describe
with pytest.raises(TypeError):
describe({"a": [1.0]})
with pytest.raises(ValueError):
describe(pd.DataFrame({"city": ["X", "Y"]}))
# ------------------------------------------------------ stats: zscore
def test_zscore_matches_manual_formula():
from mcm_toolkit.stats import zscore
x = np.array([1.0, 2.0, 3.0, 4.0])
assert np.allclose(np.asarray(zscore(x)), (x - x.mean()) / x.std())
def test_zscore_is_per_column_and_keeps_frame(rng):
from mcm_toolkit.stats import zscore
df = pd.DataFrame({"a": rng.normal(size=200), "b": rng.normal(5, 3, size=200)})
out = zscore(df)
assert isinstance(out, pd.DataFrame)
assert list(out.columns) == ["a", "b"]
assert np.allclose(np.asarray(out).mean(axis=0), 0.0, atol=1e-12)
assert np.allclose(np.asarray(out).std(axis=0), 1.0, atol=1e-12)
def test_zscore_constant_column_stays_zero():
from mcm_toolkit.stats import zscore
assert np.allclose(np.asarray(zscore([2.0, 2.0, 2.0])), 0.0)
def test_zscore_rejects_bad_type_and_all_text_frame():
from mcm_toolkit.stats import zscore
with pytest.raises(TypeError):
zscore("abc")
with pytest.raises(ValueError):
zscore(pd.DataFrame({"city": ["X"]}))
# ------------------------------------------- stats: coefficient_of_variation
def test_coefficient_of_variation_matches_manual_formula():
from mcm_toolkit.stats import coefficient_of_variation
x = np.array([1.0, 2.0, 3.0, 4.0])
assert coefficient_of_variation(x) == pytest.approx(x.std() / x.mean())
def test_coefficient_of_variation_accepts_single_column_frame():
from mcm_toolkit.stats import coefficient_of_variation
df = pd.DataFrame({"a": [1.0, 2.0, 3.0]})
single = coefficient_of_variation([1.0, 2.0, 3.0])
assert coefficient_of_variation(df) == pytest.approx(single)
def test_coefficient_of_variation_zero_mean_is_value_error():
"""均值为 0 是**取值**问题不是类型问题:必须是 ValueError 而不是 TypeError。"""
from mcm_toolkit.stats import coefficient_of_variation
with pytest.raises(ValueError):
coefficient_of_variation([-1.0, 1.0])
def test_coefficient_of_variation_rejects_empty_2d_and_bad_type():
from mcm_toolkit.stats import coefficient_of_variation
with pytest.raises(ValueError):
coefficient_of_variation([])
with pytest.raises(ValueError):
coefficient_of_variation([[1.0, 2.0], [3.0, 4.0]])
with pytest.raises(ValueError):
coefficient_of_variation(pd.DataFrame({"a": [1.0], "b": [2.0]}))
with pytest.raises(TypeError):
coefficient_of_variation(5.0)