81 lines
2.3 KiB
Python
81 lines
2.3 KiB
Python
|
"""缺失值、离群值、缩尾。
|
|||
|
|
|
|||
|
|
`detect_outliers` 属于 S 档：不只是"算出来"，还要能解释给队友听
|
|||
|
|
为什么要踢掉这几个点（论文里必须交代），所以配完整函数卡片
|
|||
|
|
（docs/dev/function-card-template.md）。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
from typing import Any, Literal
|
|||
|
|
|
|||
|
|
__all__ = ["clean_missing", "detect_outliers", "impute_missing", "winsorize"]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def clean_missing(data: Any, axis: int = 0, threshold: float | None = None) -> Any:
|
|||
|
|
"""删除缺失过多的行/列。
|
|||
|
|
|
|||
|
|
Parameters
|
|||
|
|
----------
|
|||
|
|
axis:
|
|||
|
|
0 = 按行删，1 = 按列删。
|
|||
|
|
threshold:
|
|||
|
|
缺失比例超过该值就删；`None` 表示只要有缺失就删。
|
|||
|
|
|
|||
|
|
Examples
|
|||
|
|
--------
|
|||
|
|
>>> clean_missing(df, threshold=0.5) # doctest: +SKIP
|
|||
|
|
"""
|
|||
|
|
# TODO(L0)
|
|||
|
|
raise NotImplementedError
|
|||
|
|
|
|||
|
|
|
|||
|
|
def impute_missing(data: Any, method: str = "mean", **kwargs: Any) -> Any:
|
|||
|
|
"""缺失值填补。
|
|||
|
|
|
|||
|
|
Parameters
|
|||
|
|
----------
|
|||
|
|
method:
|
|||
|
|
`"mean"` / `"median"` / `"mode"` / `"ffill"` / `"interpolate"` / `"knn"`。
|
|||
|
|
|
|||
|
|
Notes
|
|||
|
|
-----
|
|||
|
|
论文里必须说明用了哪种填补、为什么。填 `mean` 之前先想一下
|
|||
|
|
"这个缺失是不是非随机缺失（MNAR）"，否则结论可能被填补方式带偏。
|
|||
|
|
"""
|
|||
|
|
# TODO(L0)
|
|||
|
|
raise NotImplementedError
|
|||
|
|
|
|||
|
|
|
|||
|
|
def detect_outliers(data: Any, method: Literal["iqr", "zscore"] = "iqr", **kwargs: Any) -> Any:
|
|||
|
|
"""标出离群点位置（返回布尔掩码，不直接删）。
|
|||
|
|
|
|||
|
|
Parameters
|
|||
|
|
----------
|
|||
|
|
method:
|
|||
|
|
`"iqr"`：Q1-1.5·IQR 之外（`k` 可调，默认 1.5）；
|
|||
|
|
`"zscore"`：|z| > threshold（默认 3.0）。
|
|||
|
|
|
|||
|
|
Returns
|
|||
|
|
-------
|
|||
|
|
与原数据同形的布尔 DataFrame/ndarray，`True` 表示该点是离群点。
|
|||
|
|
|
|||
|
|
Notes
|
|||
|
|
-----
|
|||
|
|
**只标不删**是刻意的：删点是有代价的判断，必须由人来做，
|
|||
|
|
并且要在论文里写清楚删了几个、依据是什么。
|
|||
|
|
|
|||
|
|
Examples
|
|||
|
|
--------
|
|||
|
|
>>> mask = detect_outliers([1, 2, 3, 4, 100], method="iqr")
|
|||
|
|
>>> bool(mask[-1])
|
|||
|
|
True
|
|||
|
|
"""
|
|||
|
|
# TODO(L0) ★ S 档：配完整函数卡片
|
|||
|
|
raise NotImplementedError
|
|||
|
|
|
|||
|
|
|
|||
|
|
def winsorize(data: Any, lower: float = 0.01, upper: float = 0.99) -> Any:
|
|||
|
|
"""缩尾：把超出分位数的值压到分位点，而不是删除。"""
|
|||
|
|
# TODO(L1)
|
|||
|
|
raise NotImplementedError
|