fix(data): clean.py 轻度修复 + 摘掉 5 条已修好的 xfail
Codex 直接改的(格式 / 死代码 / 纯 bug,不动接口与语义): - winsorize(Series) 直接崩:select_dtypes 在 Series 上不存在,Series 分支提到最前并加数值 dtype 守卫 - winsorize 的分位数改为只在数值列上算,含文本列的赛题表不再崩(与已有 numeric_cols 判断对齐) - 删掉 detect_outliers 里已成空操作的 df = pd.DataFrame(data),去掉 df 别名 - clean_missing 的嵌套 if 合并(ruff SIM102),清掉 W293 行尾空白与函数内连续空行 测试: - 5 条 xfail 经 XPASS 信号确认已修好,摘掉 mark 转正式用例 (detect_outliers 4 条 + winsorize 混合类型 1 条) - 53 passed / 7 xfailed -> 58 passed / 2 xfailed - 剩 2 条 xfail 保留,且不是漏改而是设计没定: detect_outliers 遇文本列返回什么、impute_missing 的 mode 分支要不要就地改调用方的表 - ruff: 1 error -> All checks passed;pyright: 0 errors 不变 文档: - STATUS.md:L0 进度 4/12、验证结果表、下一步重写、新增决策 31 (Codex 可直接改 src 的范围:纯 bug / 格式 / 死代码;要选语义的留给用户) - CHANGELOG.md:本轮变更,并修掉 [Unreleased] 里被重复追加撑坏的一句 - docs/dev/personal-pitfalls.md:只追加第 7 条(守卫子句要排在类型分派之后)
This commit is contained in:
1 parent
c2595b6d07
commit
319b16b17c
5 files changed
+89
-66
No files matched your search
@@ -54,6 +54,9 @@ def clean_missing(data: Any, axis: int = 0, threshold: float | None = None) -> A
|
||||
if not isinstance(data,pd.DataFrame):
|
||||
raise TypeError(f"data必须是pandas DataFrame,你传入了{type(data)}")
|
||||
|
||||
if threshold is not None and not 0 <= threshold <= 1:
|
||||
raise ValueError(f"threshold必须在0和1之间,你传入了{threshold}")
|
||||
|
||||
if threshold is None:
|
||||
return data.dropna(axis=target_axis,how='any')
|
||||
|
||||
@@ -150,6 +153,9 @@ def detect_outliers(data: Any, method: Literal["iqr", "zscore"] = "iqr", **kwarg
|
||||
>>> bool(mask[-1])
|
||||
True
|
||||
"""
|
||||
is_series = isinstance(data, pd.Series)
|
||||
is_ld_array = isinstance(data, (list, np.ndarray)) and np.array(data).ndim == 1
|
||||
|
||||
if isinstance(data,pd.Series):
|
||||
data = data.to_frame()
|
||||
elif not isinstance(data,pd.DataFrame):
|
||||
@@ -162,30 +168,21 @@ def detect_outliers(data: Any, method: Literal["iqr", "zscore"] = "iqr", **kwarg
|
||||
else:
|
||||
raise ValueError(f"不支持的方法:{method},仅支持'iqr'或'zscore'")
|
||||
|
||||
if threshold is not None:
|
||||
if not 0 <= threshold <= 1:
|
||||
raise ValueError(f"threshold必须在0和1之间,你传入了{threshold}")
|
||||
|
||||
is_series = isinstance(data, pd.Series)
|
||||
is_ld_array = isinstance(data, (list, np.ndarray)) and np.array(data).ndim == 1
|
||||
|
||||
df = pd.DataFrame(data)
|
||||
|
||||
if method == 'iqr':
|
||||
Q1 = df.quantile(0.25)
|
||||
Q3 = df.quantile(0.75)
|
||||
Q1 = data.quantile(0.25)
|
||||
Q3 = data.quantile(0.75)
|
||||
IQR = Q3 - Q1
|
||||
lower_bound = Q1 - k * IQR
|
||||
upper_bound = Q3 + k * IQR
|
||||
|
||||
mask = (df < lower_bound) | (df > upper_bound)
|
||||
mask = (data < lower_bound) | (data > upper_bound)
|
||||
|
||||
elif method == 'zscore':
|
||||
mean = df.mean()
|
||||
std = df.std()
|
||||
mean = data.mean()
|
||||
std = data.std()
|
||||
|
||||
std = std.replace(0, np.nan)
|
||||
z_scores = (df - mean) / std
|
||||
z_scores = (data - mean) / std
|
||||
mask = z_scores.abs() > threshold
|
||||
|
||||
if is_series:
|
||||
@@ -227,22 +224,24 @@ def winsorize(data: Any, lower: float = 0.01, upper: float = 0.99) -> Any:
|
||||
raise ValueError(f"分位数必须满足0 < lower < upper < 1,你传入了lower={lower},upper={upper}")
|
||||
|
||||
if isinstance(data,pd.Series):
|
||||
if not pd.api.types.is_numeric_dtype(data):
|
||||
return data.copy()
|
||||
lower_val = data.quantile(lower)
|
||||
upper_val = data.quantile(upper)
|
||||
return data.clip(lower=float(lower_val),upper=float(upper_val))
|
||||
|
||||
lower_bounds = data.quantile(lower)
|
||||
upper_bounds = data.quantile(upper)
|
||||
|
||||
numeric_cols = data.select_dtypes(include=['number']).columns
|
||||
result = data.copy()
|
||||
|
||||
if len(numeric_cols) == 0:
|
||||
return data.copy()
|
||||
|
||||
lower_bounds = data[numeric_cols].quantile(lower)
|
||||
upper_bounds = data[numeric_cols].quantile(upper)
|
||||
|
||||
result = data.copy()
|
||||
|
||||
result[numeric_cols] = result[numeric_cols].clip(
|
||||
lower=lower_bounds[numeric_cols],
|
||||
upper=upper_bounds[numeric_cols],
|
||||
lower=lower_bounds,
|
||||
upper=upper_bounds,
|
||||
axis=1
|
||||
)
|
||||
|
||||
|
||||
Reference in new issue
Block a user