feat(data): clean.py 四个函数落地;补测试、docstring 与 ruff 清理
src 由用户本人写(决策 25),Codex 只做审查、补测试与文档。 实现(用户): - clean_missing:按行/列删缺失过多的行,threshold 用 math.ceil 算 min_non_na (3 列、阈值 0.5 时必须是 2;写 int() 截断会漏删) - impute_missing / detect_outliers / winsorize(原属 L1,顺手落地) Codex 改的轻度项: - 按 §4.7 给四个函数补 Parameters / Returns / Raises(与 io.py 同一套 numpydoc 格式) - ruff 6 -> 1:导入分组(I001)、elif 链改 if 加末尾无条件 return(RET505/RET503)、 去掉多余 f 前缀(F541)、删掉只会被覆盖的 mask 初始化、去掉行尾空白(W293) - tests/test_data.py:新增 13 条 clean 行为测试 + 7 条 xfail 结果: - pytest 53 passed / 42 skipped / 7 xfailed(此前 40 / 44 / 0) - pyright 0 errors, 0 warnings - ruff 剩 1 条 SIM102,就在 detect_outliers 里那段待修的代码中,删掉即全绿 已知未修(详见 STATUS.md「下一步」): - detect_outliers 三个 bug:threshold 范围校验放错函数(iqr 分支 UnboundLocalError、 zscore 默认 3.0 被 [0,1] 拒掉)、quantile 在含文本列的表上崩、 is_series / is_ld_array 在 data 已转 DataFrame 之后才判断,因此永远为 False - winsorize 同样在含文本列的表上崩 - impute_missing 的 mode 分支就地修改调用方的表
This commit is contained in:
1 parent
97a2f95066
commit
c2595b6d07
4 files changed
+414
-36
No files matched your search
+176
-5
@@ -119,21 +119,143 @@ def test_save_table_validates_before_touching_disk(tmp_path):
|
||||
assert sorted(p.name for p in tmp_path.iterdir()) == []
|
||||
|
||||
|
||||
# ------------------------------------------------- 以下为未实现算法(L0/L1 待写)
|
||||
# --------------------------------------------------------------- data: clean
|
||||
|
||||
|
||||
@pytest.mark.skip(reason=_NOT_IMPLEMENTED)
|
||||
def test_clean_missing_default_drops_any_missing():
|
||||
"""不传 threshold 时是"只要有缺失就删"。"""
|
||||
from mcm_toolkit.data import clean_missing
|
||||
|
||||
df = pd.DataFrame({"a": [1.0, np.nan, 3.0], "b": [1.0, 2.0, np.nan]})
|
||||
out = clean_missing(df)
|
||||
assert len(out) == 1
|
||||
assert list(out.columns) == ["a", "b"]
|
||||
|
||||
|
||||
def test_clean_missing_threshold_keeps_rows_within_limit():
|
||||
"""四行三列、阈值 0.5:缺 2/3 的两行要删,剩 2 行。
|
||||
|
||||
3 列时 min_non_na 必须是 ceil(3*0.5)=2;写成 int() 截断就会漏删。
|
||||
"""
|
||||
from mcm_toolkit.data import clean_missing
|
||||
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"a": [1.0, np.nan, np.nan, 4.0],
|
||||
"b": [1.0, np.nan, np.nan, 4.0],
|
||||
"c": [1.0, 2.0, np.nan, 4.0],
|
||||
}
|
||||
)
|
||||
out = clean_missing(df, axis=0, threshold=0.5)
|
||||
assert len(out) == 2
|
||||
assert out["a"].tolist() == [1.0, 4.0]
|
||||
|
||||
|
||||
def test_clean_missing_axis1_drops_empty_column():
|
||||
"""axis=1 按列删:整列全空(缺失率 1.0 > 0.5)要掉。"""
|
||||
from mcm_toolkit.data import clean_missing
|
||||
|
||||
df = pd.DataFrame({"x": [1.0, 2.0, 3.0], "y": [np.nan, np.nan, np.nan]})
|
||||
assert list(clean_missing(df, axis=1, threshold=0.5).columns) == ["x"]
|
||||
|
||||
|
||||
def test_clean_missing_rejects_bad_axis():
|
||||
from mcm_toolkit.data import clean_missing
|
||||
|
||||
with pytest.raises(ValueError, match="axis"):
|
||||
clean_missing(pd.DataFrame({"a": [1.0]}), axis=2)
|
||||
|
||||
|
||||
def test_clean_missing_rejects_non_dataframe():
|
||||
from mcm_toolkit.data import clean_missing
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
clean_missing([[1.0, 2.0]])
|
||||
|
||||
|
||||
def test_impute_missing_mean_and_ffill():
|
||||
from mcm_toolkit.data import impute_missing
|
||||
|
||||
df = pd.DataFrame({"a": [1.0, np.nan, 3.0]})
|
||||
assert impute_missing(df, method="mean")["a"].tolist() == [1.0, 2.0, 3.0]
|
||||
assert impute_missing(df, method="ffill")["a"].tolist() == [1.0, 1.0, 3.0]
|
||||
|
||||
|
||||
def test_impute_missing_mode_survives_all_nan_column():
|
||||
"""整列全 NaN 时 mode() 给不出众数:不能崩,也不能硬填一个值进去。"""
|
||||
from mcm_toolkit.data import impute_missing
|
||||
|
||||
df = pd.DataFrame({"a": [np.nan, np.nan], "b": [1.0, np.nan]})
|
||||
out = impute_missing(df, method="mode")
|
||||
assert bool(out["a"].isna().all())
|
||||
assert out["b"].tolist() == [1.0, 1.0]
|
||||
|
||||
|
||||
def test_impute_missing_rejects_unknown_method():
|
||||
from mcm_toolkit.data import impute_missing
|
||||
|
||||
with pytest.raises(ValueError, match="bogus"):
|
||||
impute_missing(pd.DataFrame({"a": [1.0]}), method="bogus")
|
||||
|
||||
|
||||
def test_impute_missing_rejects_non_dataframe():
|
||||
from mcm_toolkit.data import impute_missing
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
impute_missing([[1.0, 2.0]])
|
||||
|
||||
|
||||
def test_winsorize_clips_series_to_quantiles():
|
||||
from mcm_toolkit.data import winsorize
|
||||
|
||||
s = pd.Series([1.0, 2.0, 3.0, 4.0, 1000.0])
|
||||
out = winsorize(s, lower=0.1, upper=0.9)
|
||||
assert float(out.iloc[0]) == pytest.approx(1.4)
|
||||
assert float(out.iloc[-1]) == pytest.approx(601.6)
|
||||
assert float(out.max()) < 1000.0
|
||||
|
||||
|
||||
def test_winsorize_rejects_bad_quantile_order():
|
||||
from mcm_toolkit.data import winsorize
|
||||
|
||||
with pytest.raises(ValueError, match="lower"):
|
||||
winsorize(pd.Series([1.0, 2.0]), lower=0.9, upper=0.1)
|
||||
|
||||
|
||||
def test_winsorize_rejects_non_frame_with_typeerror():
|
||||
"""§4.7:类型不对必须抛 TypeError,不能抛 ValueError。"""
|
||||
from mcm_toolkit.data import winsorize
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
winsorize([1.0, 2.0, 3.0])
|
||||
|
||||
|
||||
def test_detect_outliers_rejects_unknown_method():
|
||||
"""方法名校验在取随机取值之前,所以这条现在就是绿的。"""
|
||||
from mcm_toolkit.data import detect_outliers
|
||||
|
||||
with pytest.raises(ValueError, match="bogus"):
|
||||
detect_outliers([1.0, 2.0], method="bogus") # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ------------------------------- 待修:契约已写进 docstring,实现还没跟上(xfail)
|
||||
#
|
||||
# 这些用例写的是**文档里已经承诺的行为**。现在跑是红的,用 xfail 让它们照样出现在
|
||||
# 报告里(显示成 x);实现修好之后会自动变成 XPASS —— 看到 XPASS 就把 mark 摘掉。
|
||||
|
||||
|
||||
@pytest.mark.xfail(reason="threshold 只在 zscore 分支赋值,iqr 走进去 UnboundLocalError", strict=False)
|
||||
def test_detect_outliers_finds_planted_points():
|
||||
"""人工埋 3 个离群点,IQR 法应正好命中 3 个。"""
|
||||
from mcm_toolkit.data import detect_outliers
|
||||
|
||||
x = np.concatenate([np.arange(100.0), [1e6, -1e6, 1e7]])
|
||||
mask = np.asarray(detect_outliers(x, method="iqr"))
|
||||
assert mask.sum() == 3
|
||||
assert mask[-3:].all()
|
||||
assert int(mask.sum()) == 3
|
||||
assert bool(mask[-3:].all())
|
||||
|
||||
|
||||
@pytest.mark.skip(reason=_NOT_IMPLEMENTED)
|
||||
@pytest.mark.xfail(reason="返回 (n,1),契约是「与原数据同形」", strict=False)
|
||||
def test_detect_outliers_returns_mask_not_filtered_data():
|
||||
"""约定:只标不删——返回与原数据同形的布尔掩码。"""
|
||||
from mcm_toolkit.data import detect_outliers
|
||||
@@ -144,6 +266,55 @@ def test_detect_outliers_returns_mask_not_filtered_data():
|
||||
assert mask.dtype == bool
|
||||
|
||||
|
||||
@pytest.mark.xfail(reason="zscore 的 threshold 被按 [0,1] 校验,默认 3.0 直接被拒", strict=False)
|
||||
def test_detect_outliers_zscore_flags_planted_point():
|
||||
from mcm_toolkit.data import detect_outliers
|
||||
|
||||
x = np.concatenate([np.zeros(99), [1e6]])
|
||||
mask = np.asarray(detect_outliers(x, method="zscore"))
|
||||
assert int(mask.sum()) == 1
|
||||
assert bool(mask[-1].all())
|
||||
|
||||
|
||||
@pytest.mark.xfail(reason="quantile 在全表上算,遇到文本列直接崩", strict=False)
|
||||
def test_detect_outliers_survives_text_columns():
|
||||
"""赛题表几乎都带文本列,分位数只能在数值列上算。"""
|
||||
from mcm_toolkit.data import detect_outliers
|
||||
|
||||
df = pd.DataFrame({"city": ["a", "b", "c", "d"], "v": [1.0, 2.0, 3.0, 1000.0]})
|
||||
assert int(np.asarray(detect_outliers(df)).sum()) == 1
|
||||
|
||||
|
||||
@pytest.mark.xfail(reason="quantile 在全表上算,遇到文本列直接崩", strict=False)
|
||||
def test_winsorize_survives_text_columns():
|
||||
from mcm_toolkit.data import winsorize
|
||||
|
||||
df = pd.DataFrame({"city": ["a", "b", "c", "d"], "v": [1.0, 2.0, 3.0, 1000.0]})
|
||||
out = winsorize(df)
|
||||
assert out["city"].tolist() == ["a", "b", "c", "d"]
|
||||
assert float(out["v"].iloc[-1]) < 1000.0
|
||||
|
||||
|
||||
@pytest.mark.xfail(reason="threshold 没做范围校验,传 1.5 会变成「什么都不删」", strict=False)
|
||||
def test_clean_missing_rejects_out_of_range_threshold():
|
||||
from mcm_toolkit.data import clean_missing
|
||||
|
||||
with pytest.raises(ValueError, match="threshold"):
|
||||
clean_missing(pd.DataFrame({"a": [1.0, np.nan]}), threshold=1.5)
|
||||
|
||||
|
||||
@pytest.mark.xfail(reason="mode 分支在调用方的表上就地改,其余分支都返回新表", strict=False)
|
||||
def test_impute_missing_mode_does_not_mutate_input():
|
||||
from mcm_toolkit.data import impute_missing
|
||||
|
||||
df = pd.DataFrame({"a": [1.0, np.nan, 1.0]})
|
||||
impute_missing(df, method="mode")
|
||||
assert bool(df["a"].isna().any())
|
||||
|
||||
|
||||
# ------------------------------------------------- 以下为未实现算法(L0/L1 待写)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason=_NOT_IMPLEMENTED)
|
||||
def test_normalize_minmax_range(rng):
|
||||
from mcm_toolkit.data import normalize
|
||||
|
||||
Reference in new issue
Block a user