feat(data): transform.py 四个函数落地;补 18 条测试、清 5 条 ruff
实现(用户本人写): - normalize(minmax / vector / max,三种都按列)+ feature_range 校验 + 常量列不出 nan - standardize(按列,ddof 真的传下去)+ 类型守卫 - log_transform(offset=None 时自动取 1 - min(x),保证 x + offset >= 1)+ 类型守卫 - one_hot_encode(自动识别对象列 / 指定列 / drop_first) - data/ 下的 L0 至此结清(L0 7/12) Codex 的轻度修复(不改接口与语义): - 清掉 5 条 ruff:RET503 / F541 / RET504 / RET505 / SIM108 - supported_method 与 feature_range 的校验提到 np.asarray 之前(守卫集中开头) - 修掉说反的报错文案(条件是要求下界 < 上界,原文写"上界必须小于下界") - one_hot_encode 自动识别时显式加上 str / string:只写 object 在 pandas 3 上是弃用路径, 每次调用发 Pandas4Warning,未来会变成"一列都选不中"的静默错误(详见坑 9) 测试: - 新写 18 条(transform 段),其中两条是把原先挂着的 skip 转为实跑 —— normalize / standardize 的实现此前一条测试都没跑到(详见个人易错点 9) - 82 passed / 40 skipped / 0 xfailed;ruff 0;pyright 0 文档: - STATUS.md:E 节 + 验证表(122 条用例分布)+ 下一步重写(transform 待定的 4 个口径)+ 决策 34 - docs/04-environment.md:新增坑 9(pandas 3 的 str dtype 与 select_dtypes 弃用路径,含实测对照表) - docs/dev/personal-pitfalls.md:只追加第 9 条(实现写完但测试还挂 skip,绿灯是假的) - CHANGELOG.md:本轮变更 + 记录 transform 待定的 4 个口径
This commit is contained in:
1 parent
5749e50e83
commit
6eab0babde
6 files changed
+330
-32
No files matched your search
@@ -10,6 +10,9 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
__all__ = ["log_transform", "normalize", "one_hot_encode", "standardize"]
|
||||
|
||||
|
||||
@@ -33,14 +36,53 @@ def normalize(x: Any, method: str = "minmax", feature_range: tuple[float, float]
|
||||
>>> normalize([1, 2, 3]).tolist()
|
||||
[0.0, 0.5, 1.0]
|
||||
"""
|
||||
# TODO(L0)
|
||||
raise NotImplementedError
|
||||
supported_method = ['minmax','vector','max']
|
||||
if method not in supported_method:
|
||||
raise ValueError(f"不支持的方法:{method},仅支持{supported_method}")
|
||||
|
||||
if method == 'minmax':
|
||||
if not isinstance(feature_range, tuple) or len(feature_range) != 2:
|
||||
raise TypeError(f"feature_range必须是形如(low, high)的二元组,你传入了{feature_range}")
|
||||
if feature_range[0] >= feature_range[1]:
|
||||
raise ValueError(f"feature_range的下界必须小于上界,你传入了{feature_range}")
|
||||
|
||||
arr = np.asarray(x,dtype=float)
|
||||
|
||||
if method == 'minmax':
|
||||
x_min = np.min(arr, axis=0)
|
||||
x_max = np.max(arr, axis=0)
|
||||
|
||||
x_range = x_max - x_min
|
||||
x_range = np.where(x_range == 0, 1.0, x_range)
|
||||
|
||||
scaled = (arr - x_min) / x_range
|
||||
return scaled * (feature_range[1] - feature_range[0]) + feature_range[0]
|
||||
|
||||
if method == 'vector':
|
||||
norm = np.linalg.norm(arr, axis=0)
|
||||
norm = np.where(norm == 0, 1.0, norm)
|
||||
return arr / norm
|
||||
|
||||
# 上面已校验 method 合法,所以这里不必再写 elif,兜住仅剩的 'max'
|
||||
x_max = np.max(arr, axis=0)
|
||||
x_max = np.where(x_max == 0, 1.0, x_max)
|
||||
return arr / x_max
|
||||
|
||||
|
||||
def standardize(x: Any, ddof: int = 0) -> Any:
|
||||
"""标准化 `(x - mean) / std`,输出均值 0、标准差 1。"""
|
||||
# TODO(L0)
|
||||
raise NotImplementedError
|
||||
if not isinstance(x, (int, float, list, tuple, np.ndarray, pd.Series)):
|
||||
raise TypeError(f"输入必须为数值型、列表、numpy array 或 pandas Series,你传入了{type(x)}")
|
||||
|
||||
arr = np.asarray(x, dtype=float)
|
||||
|
||||
mean = np.mean(arr, axis=0)
|
||||
|
||||
std = np.std(arr, axis=0, ddof=ddof)
|
||||
|
||||
std = np.where(std == 0, 1.0, std)
|
||||
|
||||
return (arr - mean) / std
|
||||
|
||||
|
||||
def log_transform(x: Any, offset: float | None = None) -> Any:
|
||||
@@ -48,11 +90,33 @@ def log_transform(x: Any, offset: float | None = None) -> Any:
|
||||
|
||||
`offset=None` 时自动取 `1 - min(x)`(含 0 或负值时)保证定义域。
|
||||
"""
|
||||
# TODO(L1)
|
||||
raise NotImplementedError
|
||||
if not isinstance(x, (int, float, list, tuple, np.ndarray, pd.Series)):
|
||||
raise TypeError(f"输入必须为数值型、列表、numpy array 或 pandas Series,你传入了{type(x)}")
|
||||
|
||||
arr = np.asarray(x, dtype=float)
|
||||
|
||||
if offset is None:
|
||||
min_val = np.min(arr)
|
||||
offset = 1.0 - min_val if min_val <= 0 else 0.0
|
||||
return np.log(arr + offset)
|
||||
|
||||
|
||||
def one_hot_encode(data: Any, columns: list[str] | None = None, drop_first: bool = False) -> Any:
|
||||
"""类别变量独热编码。"""
|
||||
# TODO(L1)
|
||||
raise NotImplementedError
|
||||
if not isinstance(data, pd.DataFrame):
|
||||
raise TypeError(f"data必须是 pandas DataFrame, 你传入了{type(data)}")
|
||||
|
||||
if columns is not None:
|
||||
missing_cols = [col for col in columns if col not in data.columns]
|
||||
if missing_cols:
|
||||
raise ValueError(f"指定的列不存在于数据中:{missing_cols}")
|
||||
cols_to_encode = columns
|
||||
else:
|
||||
# 'str' / 'string' 必须显式写上:pandas 3 起字符串列的 dtype 是 'str',
|
||||
# 只写 'object' 目前还能选中,但会发 Pandas4Warning,且官方声明未来移除该行为
|
||||
# —— 到那天这里会**静默**一列都不编码,比报错难查得多。
|
||||
object_like = ['object', 'category', 'str', 'string']
|
||||
cols_to_encode = data.select_dtypes(include=object_like).columns.tolist()
|
||||
|
||||
return pd.get_dummies(data, columns=cols_to_encode, drop_first=drop_first)
|
||||
|
||||
Reference in new issue
Block a user