Scikit-Learn Model Evaluation

Real-World Scenario

A fraud detection team builds a model that achieves 99.2% accuracy and celebrates — until they realize their dataset is 99.5% non-fraud. Their model predicts “not fraud” for everything. Proper evaluation metrics would have caught this immediately. Model evaluation is not a formality; it’s the difference between shipping a useful model and shipping a dangerous one.

Cross-Validation

from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score, StratifiedKFold, cross_validate
import numpy as np

X, y = load_breast_cancer(return_X_y=True)
model = RandomForestClassifier(n_estimators=100, random_state=42)

# 5-fold CV — splits data into 5 parts, trains on 4, tests on 1, rotates
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
scores = cross_val_score(model, X, y, cv=cv, scoring="accuracy")

print(f"Accuracy per fold: {scores.round(4)}")
print(f"Mean:  {scores.mean():.4f}")
print(f"Std:   {scores.std():.4f}")   # low std = stable model

# cross_validate — multiple metrics at once + fit time
results = cross_validate(model, X, y, cv=cv, scoring=["accuracy", "f1", "roc_auc"])
for metric in ["accuracy", "f1", "roc_auc"]:
    vals = results[f"test_{metric}"]
    print(f"{metric:12s}: {vals.mean():.4f} ± {vals.std():.4f}")

Classification Metrics

from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import (
    accuracy_score, precision_score, recall_score, f1_score,
    classification_report, confusion_matrix, roc_auc_score,
    ConfusionMatrixDisplay,
)
import numpy as np

X, y = load_breast_cancer(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)

model = GradientBoostingClassifier(random_state=42)
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
y_prob = model.predict_proba(X_test)[:, 1]   # probability of positive class

# Core metrics
print(f"Accuracy:  {accuracy_score(y_test, y_pred):.4f}")
print(f"Precision: {precision_score(y_test, y_pred):.4f}")   # of predicted +, how many are truly +
print(f"Recall:    {recall_score(y_test, y_pred):.4f}")      # of actual +, how many did we catch
print(f"F1:        {f1_score(y_test, y_pred):.4f}")          # harmonic mean of precision and recall
print(f"AUC-ROC:   {roc_auc_score(y_test, y_prob):.4f}")     # area under ROC curve

# Full report — per class breakdown
print("\nClassification Report:")
print(classification_report(y_test, y_pred, target_names=["malignant", "benign"]))

# Confusion matrix
cm = confusion_matrix(y_test, y_pred)
print("\nConfusion Matrix:")
print(cm)
# [[TN FP]
#  [FN TP]]
tn, fp, fn, tp = cm.ravel()
print(f"True Negatives:  {tn}")
print(f"False Positives: {fp}")
print(f"False Negatives: {fn}")
print(f"True Positives:  {tp}")

ROC Curve and AUC

from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve, auc
from sklearn.preprocessing import StandardScaler

X, y = load_breast_cancer(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s  = scaler.transform(X_test)

models = {
    "Logistic Regression": LogisticRegression(max_iter=1000),
    "Gradient Boosting":   GradientBoostingClassifier(random_state=42),
}

for name, model in models.items():
    model.fit(X_train_s, y_train)
    y_prob = model.predict_proba(X_test_s)[:, 1]
    fpr, tpr, thresholds = roc_curve(y_test, y_prob)
    roc_auc = auc(fpr, tpr)
    print(f"{name}: AUC = {roc_auc:.4f}")

Regression Metrics

from sklearn.datasets import fetch_california_housing
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import (
    mean_absolute_error, mean_squared_error,
    root_mean_squared_error, r2_score,
    mean_absolute_percentage_error,
)
import numpy as np

X, y = fetch_california_housing(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

model = GradientBoostingRegressor(n_estimators=200, random_state=42)
model.fit(X_train, y_train)
y_pred = model.predict(X_test)

print(f"MAE:  {mean_absolute_error(y_test, y_pred):.4f}")           # average absolute error — interpretable
print(f"RMSE: {root_mean_squared_error(y_test, y_pred):.4f}")       # penalizes large errors more than MAE
print(f"MAPE: {mean_absolute_percentage_error(y_test, y_pred):.2%}")# % error — useful when scale varies
print(f"R²:   {r2_score(y_test, y_pred):.4f}")                      # 1.0 = perfect, 0 = predicting mean

# Cross-validated RMSE
neg_mse = cross_val_score(model, X, y, cv=5, scoring="neg_mean_squared_error")
rmse_cv = np.sqrt(-neg_mse)
print(f"\nCV RMSE: {rmse_cv.mean():.4f} ± {rmse_cv.std():.4f}")

Handling Imbalanced Classes

from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
import numpy as np

# Simulate imbalanced fraud detection dataset
X, y = make_classification(
    n_samples=10000, n_features=20,
    weights=[0.99, 0.01],   # 99% class 0 (normal), 1% class 1 (fraud)
    random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)

# Option 1: class_weight='balanced' — internally reweights the loss
model_balanced = RandomForestClassifier(n_estimators=100, class_weight="balanced", random_state=42)
model_balanced.fit(X_train, y_train)
y_pred = model_balanced.predict(X_test)
print("With class_weight='balanced':")
print(classification_report(y_test, y_pred, target_names=["normal", "fraud"]))

# Option 2: Adjust decision threshold (default is 0.5)
y_prob = model_balanced.predict_proba(X_test)[:, 1]
threshold = 0.3   # lower threshold → higher recall, lower precision
y_pred_tuned = (y_prob >= threshold).astype(int)
print(f"With threshold={threshold}:")
print(classification_report(y_test, y_pred_tuned, target_names=["normal", "fraud"]))