61 lines
2.3 KiB
Python
61 lines
2.3 KiB
Python
|
"""data 包最小用法：读 → 清 → 补 → 判离群 → 缩尾。
|
|||
|
|
|
|||
|
|
每一步都对应 `tests/test_data.py` 里的一条用例；体量刻意压到最小。
|
|||
|
|
`stats.describe`（描述统计）还没实现，最后留了 TODO(L0) 的位置。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import tempfile
|
|||
|
|
from pathlib import Path
|
|||
|
|
|
|||
|
|
import numpy as np
|
|||
|
|
import pandas as pd
|
|||
|
|
|
|||
|
|
import mcm_toolkit as mcm
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main() -> None:
|
|||
|
|
mcm.set_seed(20260101)
|
|||
|
|
|
|||
|
|
# 1) 假装这是从 Excel 导出的赛题数据：带文本列、带缺失、带两个录入错误
|
|||
|
|
df = pd.DataFrame(
|
|||
|
|
{
|
|||
|
|
"city": ["A", "B", "C", "D", "E", "F"],
|
|||
|
|
"gdp": [1000.0, 1100.0, np.nan, 1300.0, 1400.0, 9999.0],
|
|||
|
|
"pop": [10.0, 11.0, np.nan, np.nan, 20.0, 15.0],
|
|||
|
|
"notes": ["ok", "ok", np.nan, "ok", np.nan, "异常"],
|
|||
|
|
}
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|||
|
|
path = Path(tmp) / "demo.csv"
|
|||
|
|
|
|||
|
|
# 2) 写盘再读回：load_table 按扩展名分派，.csv 会自动探测编码
|
|||
|
|
mcm.data.save_table(df, path)
|
|||
|
|
loaded = mcm.data.load_table(path)
|
|||
|
|
print(f"读回 {loaded.shape[0]} 行 × {loaded.shape[1]} 列")
|
|||
|
|
|
|||
|
|
# 3) 缺失太多的行先删：缺失比例 > 0.5 的走人（4 列，阈值 0.5 -> 至少要 2 个非空）
|
|||
|
|
kept = mcm.data.clean_missing(loaded, axis=0, threshold=0.5)
|
|||
|
|
print(f"按缺失比例删行：{loaded.shape[0]} -> {kept.shape[0]}")
|
|||
|
|
|
|||
|
|
# 4) 剩下的缺失做填补。mean 只作用于数值列，notes 列原样保留缺失
|
|||
|
|
filled = mcm.data.impute_missing(kept, method="mean")
|
|||
|
|
print("填补后每列缺失数：", filled.isna().sum().to_dict())
|
|||
|
|
|
|||
|
|
# 5) 找离群点：只标不删，删不删由人决定
|
|||
|
|
mask = mcm.data.detect_outliers(filled, method="iqr", k=1.5)
|
|||
|
|
print("检出离群点：")
|
|||
|
|
print(mask)
|
|||
|
|
print(f"共 {int(mask.sum().sum())} 个；notes 列恒 False（非数值列不参与统计）")
|
|||
|
|
|
|||
|
|
# 6) 不删也能用：缩尾把极端值压到分位点上，而不是丢掉整行
|
|||
|
|
w = mcm.data.winsorize(filled, lower=0.01, upper=0.99)
|
|||
|
|
print(f"gdp 最大值 {filled['gdp'].max():.1f} -> 缩尾后 {w['gdp'].max():.1f}")
|
|||
|
|
|
|||
|
|
# TODO(L0)：stats.describe(filled[["gdp", "pop"]]) 还没实现，实现后在这里加一段描述统计。
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|