07 - 模型评估与超参数调优¶

from sklearn.metrics import mean_squared_error
import numpy as np

y_true = np.array([3, -0.5, 2, 7])  # np.array创建NumPy数组
y_pred = np.array([2.5, 0.0, 2, 8])

mse = mean_squared_error(y_true, y_pred)
print(f"MSE: {mse}")

rmse = np.sqrt(mse)
print(f"RMSE: {rmse}")

from sklearn.metrics import r2_score

r2 = r2_score(y_true, y_pred)
print(f"R²: {r2}")

# 解释: 模型解释了{r2*100:.1f}%的方差

def mape(y_true, y_pred):
    # 注意: y_true 中包含0时会导致除零错误，需过滤或使用 SMAPE
    mask = y_true != 0
    return np.mean(np.abs((y_true[mask] - y_pred[mask]) / y_true[mask])) * 100

from sklearn.metrics import confusion_matrix
import seaborn as sns
import matplotlib.pyplot as plt

y_true = [1, 0, 1, 1, 0, 1]
y_pred = [1, 0, 0, 1, 0, 1]

cm = confusion_matrix(y_true, y_pred)

# 可视化
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
plt.xlabel('预测')
plt.ylabel('实际')
plt.show()

from sklearn.metrics import precision_score, recall_score, f1_score, classification_report

precision = precision_score(y_true, y_pred)
recall = recall_score(y_true, y_pred)
f1 = f1_score(y_true, y_pred)

print(f"Precision: {precision:.2f}")
print(f"Recall: {recall:.2f}")
print(f"F1: {f1:.2f}")

# 完整报告
print(classification_report(y_true, y_pred,
                          target_names=['健康', '患病']))

from sklearn.metrics import roc_curve, auc, roc_auc_score

# 需要概率输出
y_prob = [0.1, 0.4, 0.35, 0.8, 0.2, 0.9]

fpr, tpr, thresholds = roc_curve(y_true, y_prob)
roc_auc = auc(fpr, tpr)

# 或直接计算
auc_score = roc_auc_score(y_true, y_prob)

# 绘制ROC曲线
plt.figure()
plt.plot(fpr, tpr, label=f'ROC (AUC = {roc_auc:.2f})')
plt.plot([0, 1], [0, 1], 'k--')  # 随机线
plt.xlabel('FPR')
plt.ylabel('TPR')
plt.title('ROC曲线')
plt.legend()
plt.show()

from sklearn.metrics import precision_recall_curve, average_precision_score

precision, recall, thresholds = precision_recall_curve(y_true, y_prob)
ap = average_precision_score(y_true, y_prob)

plt.figure()
plt.plot(recall, precision, label=f'PR (AP = {ap:.2f})')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('PR曲线')
plt.legend()
plt.show()

from sklearn.metrics import precision_score

# 多分类
y_true_multi = [0, 1, 2, 0, 1, 2]
y_pred_multi = [0, 2, 1, 0, 0, 2]

# 宏平均
precision_macro = precision_score(y_true_multi, y_pred_multi,
                                  average='macro')

# 微平均
precision_micro = precision_score(y_true_multi, y_pred_multi,
                                  average='micro')

from sklearn.model_selection import train_test_split

# 第一次划分: 训练+验证 vs 测试
X_trainval, X_test, y_trainval, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 第二次划分: 训练 vs 验证
X_train, X_val, y_train, y_val = train_test_split(
    X_trainval, y_trainval, test_size=0.25, random_state=42
)

# 最终比例: 60% 训练, 20% 验证, 20% 测试

from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier

model = RandomForestClassifier(random_state=42)

# 5折交叉验证
scores = cross_val_score(model, X, y, cv=5, scoring='accuracy')

print(f"各折准确率: {scores}")
print(f"平均准确率: {scores.mean():.4f}")
print(f"标准差: {scores.std():.4f}")

from sklearn.model_selection import StratifiedKFold

skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

for train_idx, val_idx in skf.split(X, y):
    X_train, X_val = X[train_idx], X[val_idx]
    y_train, y_val = y[train_idx], y[val_idx]
    # 训练和评估...

from sklearn.model_selection import TimeSeriesSplit

tscv = TimeSeriesSplit(n_splits=5)

for train_idx, val_idx in tscv.split(X):
    # 注意: train_idx 在前, val_idx 在后
    X_train, X_val = X[train_idx], X[val_idx]
    # ...

from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC

# 定义参数网格
param_grid = {
    'C': [0.1, 1, 10, 100],
    'gamma': [1, 0.1, 0.01, 0.001],
    'kernel': ['rbf', 'linear']
}

# 网格搜索
grid = GridSearchCV(
    SVC(),
    param_grid,
    cv=5,
    scoring='accuracy',
    n_jobs=-1,  # 并行
    verbose=1
)

grid.fit(X_train, y_train)

# 最佳参数和模型
print(f"最佳参数: {grid.best_params_}")
print(f"最佳得分: {grid.best_score_:.4f}")
best_model = grid.best_estimator_

from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import randint, uniform

# 定义参数分布
param_distributions = {
    'C': uniform(loc=0.1, scale=99.9),       # 范围 [0.1, 100]（uniform的参数是loc和scale，非上下界）
    'gamma': uniform(loc=0.001, scale=0.999),  # 范围 [0.001, 1]
    'kernel': ['rbf', 'linear']
}

# 随机搜索
random_search = RandomizedSearchCV(
    SVC(),
    param_distributions,
    n_iter=50,  # 只尝试50种组合
    cv=5,
    random_state=42
)

random_search.fit(X_train, y_train)

# 推荐使用 Optuna（活跃维护，功能更强大）
# 注: scikit-optimize (skopt) 已停止维护，不建议在新项目中使用
import optuna

def objective(trial):
    C = trial.suggest_float('C', 0.1, 100, log=True)
    gamma = trial.suggest_float('gamma', 0.001, 1, log=True)

    model = SVC(C=C, gamma=gamma)
    score = cross_val_score(model, X_train, y_train, cv=5).mean()
    return score

study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=50)

print(f"最佳参数: {study.best_params}")
print(f"最佳分数: {study.best_value:.4f}")

param_grid = {
    'n_estimators': [100, 200, 500],      # 树的数量
    'max_depth': [None, 10, 20, 30],       # 最大深度
    'min_samples_split': [2, 5, 10],       # 分裂最小样本数
    'min_samples_leaf': [1, 2, 4],         # 叶节点最小样本数
    'max_features': ['sqrt', 'log2', None]  # 每次分裂考虑的特征数 ('auto'已弃用)
}

param_grid = {
    'max_depth': [3, 5, 7],                # 树深度
    'learning_rate': [0.01, 0.1, 0.3],     # 学习率
    'n_estimators': [100, 200, 500],       # 迭代次数
    'subsample': [0.8, 1.0],               # 样本采样比例
    'colsample_bytree': [0.8, 1.0],        # 特征采样比例
    'reg_alpha': [0, 0.1, 1],              # L1正则
    'reg_lambda': [1, 10, 100]             # L2正则
}

param_grid = {
    'hidden_layer_sizes': [(64,), (128, 64), (256, 128, 64)],
    'activation': ['relu', 'tanh'],
    'learning_rate_init': [0.001, 0.01, 0.1],
    'batch_size': [32, 64, 128],
    'alpha': [0.0001, 0.001, 0.01]         # L2正则
}

def diagnose_model(model, X_train, y_train, X_test, y_test):
    train_score = model.score(X_train, y_train)
    test_score = model.score(X_test, y_test)

    print(f"训练集准确率: {train_score:.4f}")
    print(f"测试集准确率: {test_score:.4f}")
    print(f"差距: {train_score - test_score:.4f}")

    if train_score < 0.7:
        print("\n⚠️ 高偏差 (欠拟合)")
        print("解决方案:")
        print("- 增加模型复杂度")
        print("- 减少正则化")
        print("- 添加更多特征")
    elif train_score - test_score > 0.1:
        print("\n⚠️ 高方差 (过拟合)")
        print("解决方案:")
        print("- 更多训练数据")
        print("- 增加正则化")
        print("- 简化模型")
        print("- 集成方法")
    else:
        print("\n✅ 模型良好!")

diagnose_model(model, X_train, y_train, X_test, y_test)

from sklearn.linear_model import Ridge, Lasso, ElasticNet

# L2正则
ridge = Ridge(alpha=1.0)

# L1正则
lasso = Lasso(alpha=1.0)

# 混合
elastic = ElasticNet(alpha=1.0, l1_ratio=0.5)

from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingClassifier

X_train, X_val, y_train, y_val = train_test_split(
    X, y, test_size=0.2
)

# 监控验证集性能
model = GradientBoostingClassifier(
    n_iter_no_change=10,     # 10轮没提升就停止
    validation_fraction=0.2
)

model.fit(X_train, y_train)

import torch.nn as nn

class Net(nn.Module):  # 继承nn.Module定义神经网络层
    def __init__(self):  # __init__构造方法，创建对象时自动调用
        super().__init__()  # super()调用父类方法
        self.dropout = nn.Dropout(0.5)  # 50%概率丢弃
        self.fc1 = nn.Linear(784, 128)
        self.fc2 = nn.Linear(128, 10)

    def forward(self, x):
        x = self.dropout(torch.relu(self.fc1(x)))
        x = self.fc2(x)
        return x

from torchvision import transforms

transform = transforms.Compose([
    transforms.RandomHorizontalFlip(),
    transforms.RandomRotation(15),
    transforms.ColorJitter(brightness=0.2, contrast=0.2),
    transforms.RandomAffine(0, shear=10),
    transforms.ToTensor()
])

import matplotlib.pyplot as plt
from sklearn.model_selection import learning_curve

train_sizes, train_scores, val_scores = learning_curve(
    model, X, y,
    train_sizes=np.linspace(0.1, 1.0, 10),
    cv=5,
    scoring='accuracy'
)

train_mean = train_scores.mean(axis=1)
val_mean = val_scores.mean(axis=1)

plt.figure()
plt.plot(train_sizes, train_mean, label='训练集')
plt.plot(train_sizes, val_mean, label='验证集')
plt.xlabel('训练样本数')
plt.ylabel('准确率')
plt.legend()
plt.title('学习曲线')
plt.show()

def quick_evaluate(model, X_train, y_train, X_test, y_test):
    """快速评估模型"""
    from sklearn.metrics import accuracy_score, classification_report
    import time

    # 训练时间
    start = time.time()
    model.fit(X_train, y_train)
    train_time = time.time() - start

    # 预测时间
    start = time.time()
    y_pred = model.predict(X_test)
    predict_time = time.time() - start

    # 性能
    train_acc = model.score(X_train, y_train)
    test_acc = accuracy_score(y_test, y_pred)

    print(f"训练时间: {train_time:.2f}s")
    print(f"预测时间: {predict_time:.4f}s")
    print(f"训练准确率: {train_acc:.4f}")
    print(f"测试准确率: {test_acc:.4f}")
    print(f"过拟合程度: {train_acc - test_acc:.4f}")
    print("\n分类报告:")
    print(classification_report(y_test, y_pred))

    return {
        'train_time': train_time,
        'predict_time': predict_time,
        'train_acc': train_acc,
        'test_acc': test_acc,
        'overfitting': train_acc - test_acc
    }

# 使用
results = quick_evaluate(model, X_train, y_train, X_test, y_test)