12 - 集成学习进阶¶

from sklearn.ensemble import BaggingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split

# 创建数据
X, y = make_classification(n_samples=1000, n_features=20, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Bagging
bagging = BaggingClassifier(
    estimator=DecisionTreeClassifier(),  # 基学习器
    n_estimators=100,                     # 基学习器数量
    max_samples=0.8,                      # 每次采样比例
    max_features=0.8,                     # 特征采样比例
    bootstrap=True,                       # 有放回采样
    bootstrap_features=False,             # 特征是否有放回
    oob_score=True,                       # 启用OOB估计
    n_jobs=-1,                            # 并行
    random_state=42
)

bagging.fit(X_train, y_train)
print(f"训练准确率: {bagging.score(X_train, y_train):.4f}")
print(f"测试准确率: {bagging.score(X_test, y_test):.4f}")
print(f"OOB分数: {bagging.oob_score_:.4f}")

import matplotlib.pyplot as plt
import numpy as np

# 训练随机森林
from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X_train, y_train)

# 获取特征重要性
importances = rf.feature_importances_
indices = np.argsort(importances)[::-1]

# 可视化
plt.figure(figsize=(10, 6))
plt.bar(range(X.shape[1]), importances[indices])
plt.title("特征重要性")
plt.xlabel("特征")
plt.ylabel("重要性")
plt.show()

# 打印前10重要特征
print("Top 10 重要特征:")
for i in range(10):
    print(f"特征 {indices[i]}: {importances[indices[i]]:.4f}")

from sklearn.model_selection import GridSearchCV

param_grid = {
    'n_estimators': [100, 200, 500],
    'max_depth': [None, 10, 20, 30],
    'min_samples_split': [2, 5, 10],
    'min_samples_leaf': [1, 2, 4],
    'max_features': ['sqrt', 'log2', None]
}

grid_search = GridSearchCV(
    RandomForestClassifier(random_state=42),
    param_grid,
    cv=5,
    n_jobs=-1,
    scoring='accuracy'
)

grid_search.fit(X_train, y_train)
print(f"最佳参数: {grid_search.best_params_}")
print(f"最佳分数: {grid_search.best_score_:.4f}")

from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier

# AdaBoost
ada = AdaBoostClassifier(
    estimator=DecisionTreeClassifier(max_depth=1),  # 决策树桩
    n_estimators=100,
    learning_rate=0.1,
    algorithm='SAMME.R',  # 或 'SAMME'
    random_state=42
)

ada.fit(X_train, y_train)
print(f"训练准确率: {ada.score(X_train, y_train):.4f}")
print(f"测试准确率: {ada.score(X_test, y_test):.4f}")

# 查看每个基学习器的权重
print(f"\n前10个基学习器权重: {ada.estimator_weights_[:10]}")

from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor

# 分类
gb_clf = GradientBoostingClassifier(
    n_estimators=100,        # 树的数量
    learning_rate=0.1,       # 学习率（收缩率）
    max_depth=3,             # 树深度
    min_samples_split=2,     # 节点分裂最小样本
    subsample=0.8,           # 子采样比例（随机梯度提升）
    random_state=42
)

gb_clf.fit(X_train, y_train)
print(f"GBDT分类准确率: {gb_clf.score(X_test, y_test):.4f}")

# 回归示例
from sklearn.datasets import make_regression
X_reg, y_reg = make_regression(n_samples=1000, n_features=10, noise=0.1)
X_train_r, X_test_r, y_train_r, y_test_r = train_test_split(X_reg, y_reg, test_size=0.2, random_state=42)

gb_reg = GradientBoostingRegressor(n_estimators=100, max_depth=3, random_state=42)
gb_reg.fit(X_train_r, y_train_r)
print(f"GBDT回归R²: {gb_reg.score(X_test_r, y_test_r):.4f}")

import xgboost as xgb
from sklearn.datasets import load_breast_cancer

# 加载数据
data = load_breast_cancer()
X, y = data.data, data.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# XGBoost分类
xgb_clf = xgb.XGBClassifier(
    n_estimators=100,
    max_depth=5,
    learning_rate=0.1,
    subsample=0.8,
    colsample_bytree=0.8,
    reg_alpha=0.1,        # L1正则
    reg_lambda=1.0,       # L2正则
    random_state=42,
    eval_metric='logloss'
)

xgb_clf.fit(X_train, y_train)
print(f"XGBoost准确率: {xgb_clf.score(X_test, y_test):.4f}")

# 特征重要性
xgb.plot_importance(xgb_clf, max_num_features=10)
plt.show()

# 转换为DMatrix（XGBoost高效数据结构）
dtrain = xgb.DMatrix(X_train, label=y_train)
dtest = xgb.DMatrix(X_test, label=y_test)

# 参数
params = {
    'objective': 'binary:logistic',
    'max_depth': 5,
    'learning_rate': 0.1,
    'subsample': 0.8,
    'colsample_bytree': 0.8,
    'eval_metric': 'auc'
}

# 训练
model = xgb.train(
    params,
    dtrain,
    num_boost_round=100,
    evals=[(dtrain, 'train'), (dtest, 'test')],
    early_stopping_rounds=10
)

# 预测
preds = model.predict(dtest)

from sklearn.model_selection import RandomizedSearchCV

param_dist = {
    'n_estimators': [100, 200, 500],
    'max_depth': [3, 5, 7, 10],
    'learning_rate': [0.01, 0.05, 0.1, 0.3],
    'subsample': [0.6, 0.8, 1.0],
    'colsample_bytree': [0.6, 0.8, 1.0],
    'reg_alpha': [0, 0.1, 1],
    'reg_lambda': [1, 10, 100]
}

random_search = RandomizedSearchCV(
    xgb.XGBClassifier(random_state=42, eval_metric='logloss'),
    param_distributions=param_dist,
    n_iter=50,
    cv=5,
    n_jobs=-1,
    random_state=42
)

random_search.fit(X_train, y_train)
print(f"最佳参数: {random_search.best_params_}")

import lightgbm as lgb

# LightGBM分类
lgb_clf = lgb.LGBMClassifier(
    n_estimators=100,
    max_depth=-1,           # -1表示无限制
    learning_rate=0.1,
    num_leaves=31,          # 最大叶子数
    subsample=0.8,
    colsample_bytree=0.8,
    reg_alpha=0.1,
    reg_lambda=1.0,
    random_state=42
)

lgb_clf.fit(X_train, y_train)
print(f"LightGBM准确率: {lgb_clf.score(X_test, y_test):.4f}")

# 原生接口
train_data = lgb.Dataset(X_train, label=y_train)
valid_data = lgb.Dataset(X_test, label=y_test, reference=train_data)

params = {
    'objective': 'binary',
    'metric': 'auc',
    'boosting_type': 'gbdt',
    'num_leaves': 31,
    'learning_rate': 0.05,
    'feature_fraction': 0.9,
    'bagging_fraction': 0.8,
    'bagging_freq': 5,
    'verbose': 0
}

model = lgb.train(
    params,
    train_data,
    num_boost_round=1000,
    valid_sets=[train_data, valid_data],
    callbacks=[lgb.early_stopping(50), lgb.log_evaluation(100)]
)

from catboost import CatBoostClassifier

# 假设有类别特征
cat_features = [0, 2, 5]  # 类别特征的列索引

cat_clf = CatBoostClassifier(
    iterations=100,
    learning_rate=0.1,
    depth=6,
    cat_features=cat_features,
    random_seed=42,
    verbose=False
)

cat_clf.fit(X_train, y_train)
print(f"CatBoost准确率: {cat_clf.score(X_test, y_test):.4f}")

from sklearn.ensemble import StackingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
import xgboost as xgb

# 定义基学习器
estimators = [
    ('rf', RandomForestClassifier(n_estimators=100, random_state=42)),
    ('xgb', xgb.XGBClassifier(n_estimators=100, random_state=42, eval_metric='logloss')),
    ('svc', SVC(probability=True, random_state=42))
]

# Stacking
stacking = StackingClassifier(
    estimators=estimators,
    final_estimator=LogisticRegression(),  # 元学习器
    cv=5,                                   # 交叉验证折数
    stack_method='predict_proba',          # 使用概率作为元特征
    n_jobs=-1
)

stacking.fit(X_train, y_train)
print(f"Stacking准确率: {stacking.score(X_test, y_test):.4f}")

# 查看各基学习器性能
for name, estimator in estimators:
    estimator.fit(X_train, y_train)
    score = estimator.score(X_test, y_test)
    print(f"{name}: {score:.4f}")

from sklearn.ensemble import VotingClassifier

# 硬投票
voting_hard = VotingClassifier(
    estimators=estimators,
    voting='hard',
    n_jobs=-1
)

voting_hard.fit(X_train, y_train)
print(f"硬投票准确率: {voting_hard.score(X_test, y_test):.4f}")

# 软投票（需要基学习器支持predict_proba）
voting_soft = VotingClassifier(
    estimators=estimators,
    voting='soft',
    weights=[1, 2, 1],  # 可以给不同模型不同权重
    n_jobs=-1
)

voting_soft.fit(X_train, y_train)
print(f"软投票准确率: {voting_soft.score(X_test, y_test):.4f}")

# XGBoost调参顺序
# 1. 确定学习率和树数量
# 2. 调树深度和叶子数
# 3. 调正则化参数
# 4. 调采样参数

# LightGBM关键参数
params = {
    'learning_rate': 0.05,      # 先调
    'num_leaves': 31,           # 控制复杂度
    'max_depth': -1,            # 与num_leaves配合
    'min_child_samples': 20,    # 防止过拟合
    'subsample': 0.8,           # 行采样
    'colsample_bytree': 0.8,    # 列采样
    'reg_alpha': 0.1,           # L1
    'reg_lambda': 1.0           # L2
}