feat(data): clean.py 四个函数落地;补测试、docstring 与 ruff 清理

src 由用户本人写(决策 25),Codex 只做审查、补测试与文档。

实现(用户):
- clean_missing:按行/列删缺失过多的行,threshold 用 math.ceil 算 min_non_na
  (3 列、阈值 0.5 时必须是 2;写 int() 截断会漏删)
- impute_missing / detect_outliers / winsorize(原属 L1,顺手落地)

Codex 改的轻度项:
- 按 §4.7 给四个函数补 Parameters / Returns / Raises(与 io.py 同一套 numpydoc 格式)
- ruff 6 -> 1:导入分组(I001)、elif 链改 if 加末尾无条件 return(RET505/RET503)、
  去掉多余 f 前缀(F541)、删掉只会被覆盖的 mask 初始化、去掉行尾空白(W293)
- tests/test_data.py:新增 13 条 clean 行为测试 + 7 条 xfail

结果:
- pytest 53 passed / 42 skipped / 7 xfailed(此前 40 / 44 / 0)
- pyright 0 errors, 0 warnings
- ruff 剩 1 条 SIM102,就在 detect_outliers 里那段待修的代码中,删掉即全绿

已知未修(详见 STATUS.md「下一步」):
- detect_outliers 三个 bug:threshold 范围校验放错函数(iqr 分支 UnboundLocalError、
  zscore 默认 3.0 被 [0,1] 拒掉)、quantile 在含文本列的表上崩、
  is_series / is_ld_array 在 data 已转 DataFrame 之后才判断,因此永远为 False
- winsorize 同样在含文本列的表上崩
- impute_missing 的 mode 分支就地修改调用方的表
This commit is contained in:
guzhujushi committed 2026-10-02 15:04:13 +08:00
1 parent 97a2f95066
commit c2595b6d07
4 files changed
+414 -36

No files matched your search

+176 -5
View File
@@ -119,21 +119,143 @@ def test_save_table_validates_before_touching_disk(tmp_path):
assert sorted(p.name for p in tmp_path.iterdir()) == []
# ------------------------------------------------- 以下为未实现算法(L0/L1 待写)
# --------------------------------------------------------------- data: clean
@pytest.mark.skip(reason=_NOT_IMPLEMENTED)
def test_clean_missing_default_drops_any_missing():
"""不传 threshold 时是"只要有缺失就删"。"""
from mcm_toolkit.data import clean_missing
df = pd.DataFrame({"a": [1.0, np.nan, 3.0], "b": [1.0, 2.0, np.nan]})
out = clean_missing(df)
assert len(out) == 1
assert list(out.columns) == ["a", "b"]
def test_clean_missing_threshold_keeps_rows_within_limit():
"""四行三列、阈值 0.5:缺 2/3 的两行要删,剩 2 行。
3 列时 min_non_na 必须是 ceil(3*0.5)=2;写成 int() 截断就会漏删。
"""
from mcm_toolkit.data import clean_missing
df = pd.DataFrame(
{
"a": [1.0, np.nan, np.nan, 4.0],
"b": [1.0, np.nan, np.nan, 4.0],
"c": [1.0, 2.0, np.nan, 4.0],
}
)
out = clean_missing(df, axis=0, threshold=0.5)
assert len(out) == 2
assert out["a"].tolist() == [1.0, 4.0]
def test_clean_missing_axis1_drops_empty_column():
"""axis=1 按列删:整列全空(缺失率 1.0 > 0.5)要掉。"""
from mcm_toolkit.data import clean_missing
df = pd.DataFrame({"x": [1.0, 2.0, 3.0], "y": [np.nan, np.nan, np.nan]})
assert list(clean_missing(df, axis=1, threshold=0.5).columns) == ["x"]
def test_clean_missing_rejects_bad_axis():
from mcm_toolkit.data import clean_missing
with pytest.raises(ValueError, match="axis"):
clean_missing(pd.DataFrame({"a": [1.0]}), axis=2)
def test_clean_missing_rejects_non_dataframe():
from mcm_toolkit.data import clean_missing
with pytest.raises(TypeError):
clean_missing([[1.0, 2.0]])
def test_impute_missing_mean_and_ffill():
from mcm_toolkit.data import impute_missing
df = pd.DataFrame({"a": [1.0, np.nan, 3.0]})
assert impute_missing(df, method="mean")["a"].tolist() == [1.0, 2.0, 3.0]
assert impute_missing(df, method="ffill")["a"].tolist() == [1.0, 1.0, 3.0]
def test_impute_missing_mode_survives_all_nan_column():
"""整列全 NaN 时 mode() 给不出众数:不能崩,也不能硬填一个值进去。"""
from mcm_toolkit.data import impute_missing
df = pd.DataFrame({"a": [np.nan, np.nan], "b": [1.0, np.nan]})
out = impute_missing(df, method="mode")
assert bool(out["a"].isna().all())
assert out["b"].tolist() == [1.0, 1.0]
def test_impute_missing_rejects_unknown_method():
from mcm_toolkit.data import impute_missing
with pytest.raises(ValueError, match="bogus"):
impute_missing(pd.DataFrame({"a": [1.0]}), method="bogus")
def test_impute_missing_rejects_non_dataframe():
from mcm_toolkit.data import impute_missing
with pytest.raises(TypeError):
impute_missing([[1.0, 2.0]])
def test_winsorize_clips_series_to_quantiles():
from mcm_toolkit.data import winsorize
s = pd.Series([1.0, 2.0, 3.0, 4.0, 1000.0])
out = winsorize(s, lower=0.1, upper=0.9)
assert float(out.iloc[0]) == pytest.approx(1.4)
assert float(out.iloc[-1]) == pytest.approx(601.6)
assert float(out.max()) < 1000.0
def test_winsorize_rejects_bad_quantile_order():
from mcm_toolkit.data import winsorize
with pytest.raises(ValueError, match="lower"):
winsorize(pd.Series([1.0, 2.0]), lower=0.9, upper=0.1)
def test_winsorize_rejects_non_frame_with_typeerror():
"""§4.7:类型不对必须抛 TypeError,不能抛 ValueError。"""
from mcm_toolkit.data import winsorize
with pytest.raises(TypeError):
winsorize([1.0, 2.0, 3.0])
def test_detect_outliers_rejects_unknown_method():
"""方法名校验在取随机取值之前,所以这条现在就是绿的。"""
from mcm_toolkit.data import detect_outliers
with pytest.raises(ValueError, match="bogus"):
detect_outliers([1.0, 2.0], method="bogus") # type: ignore[arg-type]
# ------------------------------- 待修:契约已写进 docstring,实现还没跟上(xfail)
#
# 这些用例写的是**文档里已经承诺的行为**。现在跑是红的,用 xfail 让它们照样出现在
# 报告里(显示成 x);实现修好之后会自动变成 XPASS —— 看到 XPASS 就把 mark 摘掉。
@pytest.mark.xfail(reason="threshold 只在 zscore 分支赋值,iqr 走进去 UnboundLocalError", strict=False)
def test_detect_outliers_finds_planted_points():
"""人工埋 3 个离群点,IQR 法应正好命中 3 个。"""
from mcm_toolkit.data import detect_outliers
x = np.concatenate([np.arange(100.0), [1e6, -1e6, 1e7]])
mask = np.asarray(detect_outliers(x, method="iqr"))
assert mask.sum() == 3
assert mask[-3:].all()
assert int(mask.sum()) == 3
assert bool(mask[-3:].all())
@pytest.mark.skip(reason=_NOT_IMPLEMENTED)
@pytest.mark.xfail(reason="返回 (n,1),契约是「与原数据同形」", strict=False)
def test_detect_outliers_returns_mask_not_filtered_data():
"""约定:只标不删——返回与原数据同形的布尔掩码。"""
from mcm_toolkit.data import detect_outliers
@@ -144,6 +266,55 @@ def test_detect_outliers_returns_mask_not_filtered_data():
assert mask.dtype == bool
@pytest.mark.xfail(reason="zscore 的 threshold 被按 [0,1] 校验,默认 3.0 直接被拒", strict=False)
def test_detect_outliers_zscore_flags_planted_point():
from mcm_toolkit.data import detect_outliers
x = np.concatenate([np.zeros(99), [1e6]])
mask = np.asarray(detect_outliers(x, method="zscore"))
assert int(mask.sum()) == 1
assert bool(mask[-1].all())
@pytest.mark.xfail(reason="quantile 在全表上算,遇到文本列直接崩", strict=False)
def test_detect_outliers_survives_text_columns():
"""赛题表几乎都带文本列,分位数只能在数值列上算。"""
from mcm_toolkit.data import detect_outliers
df = pd.DataFrame({"city": ["a", "b", "c", "d"], "v": [1.0, 2.0, 3.0, 1000.0]})
assert int(np.asarray(detect_outliers(df)).sum()) == 1
@pytest.mark.xfail(reason="quantile 在全表上算,遇到文本列直接崩", strict=False)
def test_winsorize_survives_text_columns():
from mcm_toolkit.data import winsorize
df = pd.DataFrame({"city": ["a", "b", "c", "d"], "v": [1.0, 2.0, 3.0, 1000.0]})
out = winsorize(df)
assert out["city"].tolist() == ["a", "b", "c", "d"]
assert float(out["v"].iloc[-1]) < 1000.0
@pytest.mark.xfail(reason="threshold 没做范围校验,传 1.5 会变成「什么都不删」", strict=False)
def test_clean_missing_rejects_out_of_range_threshold():
from mcm_toolkit.data import clean_missing
with pytest.raises(ValueError, match="threshold"):
clean_missing(pd.DataFrame({"a": [1.0, np.nan]}), threshold=1.5)
@pytest.mark.xfail(reason="mode 分支在调用方的表上就地改,其余分支都返回新表", strict=False)
def test_impute_missing_mode_does_not_mutate_input():
from mcm_toolkit.data import impute_missing
df = pd.DataFrame({"a": [1.0, np.nan, 1.0]})
impute_missing(df, method="mode")
assert bool(df["a"].isna().any())
# ------------------------------------------------- 以下为未实现算法(L0/L1 待写)
@pytest.mark.skip(reason=_NOT_IMPLEMENTED)
def test_normalize_minmax_range(rng):
from mcm_toolkit.data import normalize