Какие знаешь общие методы борьбы с переобучением линейных моделей?

Борьба с переобучением в линейных моделях

Линейные модели простые но могут быстро переучиться на высокомерных данных. Есть много способов контролировать сложность.

1. Регуляризация (Regularization)

Добавляем штраф за сложность модели к loss функции.

Ridge (L2 регуляризация)

from sklearn.linear_model import Ridge, RidgeCV
import numpy as np

# Ridge regression
model = Ridge(alpha=1.0)  # alpha контролирует силу регуляризации
model.fit(X_train, y_train)

train_score = model.score(X_train, y_train)
val_score = model.score(X_val, y_val)

print(f"Train: {train_score:.4f}, Val: {val_score:.4f}")

# Найти оптимальный alpha
model_cv = RidgeCV(alphas=[0.1, 1, 10, 100, 1000])
model_cv.fit(X_train, y_train)

print(f"Best alpha: {model_cv.alpha_}")
print(f"Coefficients are smaller (penalized): {model_cv.coef_}")

# Loss function with L2:
# Loss = MSE + alpha * sum(coef^2)
# Штрафует большие coefficients

Преимущества Ridge:

• Уменьшает все коэффициенты proportionally
• Хорошо когда много relatable признаков
• Всегда стабильна (unique решение)

Когда использовать: Много признаков, хорошие все

Lasso (L1 регуляризация)

from sklearn.linear_model import Lasso, LassoCV

# Lasso regression
model = Lasso(alpha=0.1)
model.fit(X_train, y_train)

# Loss function:
# Loss = MSE + alpha * sum(|coef|)
# Обнуляет некритичные coefficients

print(f"Number of non-zero coefficients: {np.sum(model.coef_ != 0)}")
print(f"Non-zero coefficients: {model.coef_[model.coef_ != 0]}")

# LassoCV для подбора alpha
model_cv = LassoCV(alphas=np.logspace(-4, 1, 100), cv=5)
model_cv.fit(X_train, y_train)

print(f"Best alpha: {model_cv.alpha_}")
print(f"Number of selected features: {np.sum(model_cv.coef_ != 0)}")

Преимущества Lasso:

• Автоматическая feature selection (обнулядет)
• Интерпретируется легче
• Sparse решение

Когда использовать: Много признаков, нужна интерпретация, feature selection

Elastic Net (комбинация L1 и L2)

from sklearn.linear_model import ElasticNet, ElasticNetCV

# Комбинирует преимущества Ridge и Lasso
model = ElasticNet(alpha=0.1, l1_ratio=0.5)  # 50% L1, 50% L2
model.fit(X_train, y_train)

# Loss = MSE + alpha * (l1_ratio * |coef| + (1-l1_ratio) * coef^2)

# ElasticNetCV для подбора обоих параметров
model_cv = ElasticNetCV(
    alphas=np.logspace(-3, 1, 50),
    l1_ratio=np.linspace(0, 1, 11),
    cv=5
)
model_cv.fit(X_train, y_train)

print(f"Best alpha: {model_cv.alpha_}")
print(f"Best l1_ratio: {model_cv.l1_ratio_}")

2. Early Stopping

Для iterative методов (SGD).

from sklearn.linear_model import SGDRegressor

# SGD с early stopping
model = SGDRegressor(
    loss='squared_error',
    penalty='l2',
    alpha=0.0001,
    early_stopping=True,  # early stop on validation score
    validation_fraction=0.2,  # 20% для validation
    n_iter_no_change=10,  # stop if no improvement for 10 iterations
    random_state=42
)

model.fit(X_train, y_train)
print(f"Model converged: {model.n_iter_}")

3. Feature Selection

Уменьшаем количество признаков.

SelectKBest

from sklearn.feature_selection import SelectKBest, f_regression

# Выбираем k лучших признаков
selector = SelectKBest(f_regression, k=10)
X_train_selected = selector.fit_transform(X_train, y_train)
X_val_selected = selector.transform(X_val)

model = Ridge()
model.fit(X_train_selected, y_train)
val_score = model.score(X_val_selected, y_val)

print(f"Selected {selector.get_support().sum()} features")
print(f"Val score: {val_score:.4f}")

Recursive Feature Elimination (RFE)

from sklearn.feature_selection import RFE

# Рекурсивно удаляем неважные признаки
model = Ridge()
rfe = RFE(model, n_features_to_select=10, step=1)
rfe.fit(X_train, y_train)

X_train_rfe = rfe.transform(X_train)
X_val_rfe = rfe.transform(X_val)

model_final = Ridge()
model_final.fit(X_train_rfe, y_train)
val_score = model_final.score(X_val_rfe, y_val)

print(f"Selected features: {np.where(rfe.support_)[0]}")

Lasso для selection

# Lasso автоматически выбирает features
model = LassoCV()
model.fit(X_train, y_train)

selected_features = np.where(model.coef_ != 0)[0]
print(f"Lasso selected {len(selected_features)} features")

4. Dimensionality Reduction

Уменьшаем размерность пространства признаков.

PCA

from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline

# PCA pipeline
pipeline = Pipeline([
    ('pca', PCA(n_components=10)),
    ('ridge', Ridge())
])

pipeline.fit(X_train, y_train)
val_score = pipeline.score(X_val, y_val)

print(f"Explained variance: {pipeline.named_steps['pca'].explained_variance_ratio_.sum():.4f}")

# Выбираем n_components который объясняет 95% дисперсии
pca = PCA(n_components=0.95)
X_train_pca = pca.fit_transform(X_train)
print(f"Kept {X_train_pca.shape[1]} components from {X_train.shape[1]}")

5. Cross-Validation для выбора модели

from sklearn.model_selection import cross_val_score, KFold

# Сравниваем разные регуляризационные подходы
models = [
    ('Ridge (alpha=0.1)', Ridge(alpha=0.1)),
    ('Ridge (alpha=1)', Ridge(alpha=1)),
    ('Ridge (alpha=10)', Ridge(alpha=10)),
    ('Lasso (alpha=0.1)', Lasso(alpha=0.1)),
    ('Lasso (alpha=1)', Lasso(alpha=1)),
    ('ElasticNet', ElasticNet(alpha=0.1, l1_ratio=0.5)),
]

kfold = KFold(n_splits=5, shuffle=True, random_state=42)

for name, model in models:
    scores = cross_val_score(model, X_train, y_train, cv=kfold, scoring='r2')
    print(f"{name:20}: {scores.mean():.4f} (+/- {scores.std():.4f})")

6. Polynomial Features Control

from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import Pipeline

# Не добавляй много polynomial features без регуляризации
for degree in [1, 2, 3, 4, 5]:
    pipeline = Pipeline([
        ('poly', PolynomialFeatures(degree=degree)),
        ('ridge', Ridge(alpha=1))  # обязательно добавь регуляризацию
    ])
    
    pipeline.fit(X_train, y_train)
    train_score = pipeline.score(X_train, y_train)
    val_score = pipeline.score(X_val, y_val)
    
    print(f"Degree {degree}: Train={train_score:.4f}, Val={val_score:.4f}")

7. Scaling and Normalization

from sklearn.preprocessing import StandardScaler

# Обязательно scale признаки перед регуляризацией
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_val_scaled = scaler.transform(X_val)

model = Ridge(alpha=1.0)
model.fit(X_train_scaled, y_train)
val_score = model.score(X_val_scaled, y_val)

# Без scaling регуляризация работает неправильно

8. Practical Pipeline

def build_regularized_linear_model(X_train, y_train, X_val, y_val, X_test, y_test):
    """
    Best practice pipeline для линейных моделей
    """
    
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import StandardScaler
    from sklearn.linear_model import RidgeCV, LassoCV, ElasticNetCV
    
    # 1. Try different regularization approaches
    models = {
        'Ridge': Pipeline([
            ('scaler', StandardScaler()),
            ('ridge', RidgeCV(alphas=np.logspace(-3, 3, 100)))
        ]),
        'Lasso': Pipeline([
            ('scaler', StandardScaler()),
            ('lasso', LassoCV(alphas=np.logspace(-3, 1, 100)))
        ]),
        'ElasticNet': Pipeline([
            ('scaler', StandardScaler()),
            ('elasticnet', ElasticNetCV(
                alphas=np.logspace(-3, 1, 50),
                l1_ratio=np.linspace(0.1, 0.9, 9)
            ))
        ])
    }
    
    results = {}
    
    for name, model in models.items():
        # Cross-validation на train set
        cv_scores = cross_val_score(model, X_train, y_train, cv=5, scoring='r2')
        
        # Fit на train set
        model.fit(X_train, y_train)
        
        # Evaluate
        train_score = model.score(X_train, y_train)
        val_score = model.score(X_val, y_val)
        test_score = model.score(X_test, y_test)
        
        results[name] = {
            'cv_mean': cv_scores.mean(),
            'cv_std': cv_scores.std(),
            'train_score': train_score,
            'val_score': val_score,
            'test_score': test_score,
            'model': model
        }
        
        print(f"\n{name}:")
        print(f"  CV: {cv_scores.mean():.4f} (+/- {cv_scores.std():.4f})")
        print(f"  Train: {train_score:.4f}, Val: {val_score:.4f}, Test: {test_score:.4f}")
    
    # 2. Choose best model
    best_model_name = max(results, key=lambda x: results[x]['val_score'])
    best_model = results[best_model_name]['model']
    
    print(f"\nBest model: {best_model_name}")
    
    return best_model

Сравнение методов

Golden Rules

1. Всегда используй регуляризацию на высокомерных данных
2. Scale признаки перед регуляризацией
3. Ridge - default выбор
4. Lasso - когда нужна feature selection и интерпретируемость
5. ElasticNet - best of both worlds
6. CV для выбора alpha - не гадай
7. Check train/val gap - индикатор переобучения
8. Simple is better - проще модель > сложнее регуляризация