MLCancerResearch/evaluation.py at main · rafipatel/MLCancerResearch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
# evaluation/evaluation.py
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
import numpy as np
import torch
import pandas as pd
import os


def evaluate_model(model, X_train, y_train, X_test, y_test, model_name, plot=True):

    if model_name=="nn":
        X_train = torch.FloatTensor(X_train.values)

        X_test = torch.FloatTensor(X_test.values)

        # Ensure the model is in evaluation mode if necessary
        model.eval()

        # Get predictions from the model on the training data
        with torch.no_grad():  # Ensuring no gradients are being computed
            # y_train_pred = model(X_train)
            y_train_pred = model(X_train).cpu().detach().numpy()
            y_pred = model(X_test).cpu().detach().numpy()

    else:
        # Make predictions on the testing data
        y_pred = model.predict(X_test)
        # prediction on training data
        y_train_pred = model.predict(X_train)
        if plot:
            plot_learning_curve(model, pd.concat([X_train, X_test]), pd.concat([y_train, y_test]))

    train_results = train_evaluate(y_train, y_train_pred)
    test_results = test_evaluate(y_test, y_pred)
    return train_results, test_results

def train_evaluate(y_train, y_train_pred):

        LR_train_mse_1 = mean_squared_error(y_train.values[:,0], y_train_pred[:,0])
        LR_train_mse_2 = mean_squared_error(y_train.values[:,1], y_train_pred[:,1])

    # Calculating the mean absolute error of the predictions
        LR_train_mae_1 = mean_absolute_error(y_train.values[:,0], y_train_pred[:,0])
        LR_train_mae_2 = mean_absolute_error(y_train.values[:,1], y_train_pred[:,1])

    # Calculating the root mean squared error of the predictions
        LR_train_rmse_1 = np.sqrt(LR_train_mse_1)
        LR_train_rmse_2 = np.sqrt(LR_train_mse_2)

        LR_train_Rsquared_1 = r2_score(y_train.values[:,0], y_train_pred[:,0])
        LR_train_Rsquared_2 = r2_score(y_train.values[:,1], y_train_pred[:,1])

        return [LR_train_mse_1,LR_train_mse_2,LR_train_mae_1,LR_train_mae_2,LR_train_rmse_1,LR_train_rmse_2,LR_train_Rsquared_1, LR_train_Rsquared_2 ]


# Calculating the mean squared error of the predictions
def test_evaluate(y_test,y_pred):
    LR_test_mse_1 = mean_squared_error(y_test.values[:,0], y_pred[:,0])
    LR_test_mse_2 = mean_squared_error(y_test.values[:,1], y_pred[:,1])

# Calculating the mean absolute error of the predictions
    LR_test_mae_1 = mean_absolute_error(y_test.values[:,0], y_pred[:,0])
    LR_test_mae_2 = mean_absolute_error(y_test.values[:,1], y_pred[:,1])

# Calculating the root mean squared error of the predictions
    LR_test_rmse_1 = np.sqrt(LR_test_mse_1)
    LR_test_rmse_2 = np.sqrt(LR_test_mse_2)


    # print(r2_score(y_test.values[:,0], y_pred[:,0]))
    LR_test_Rsquared_1 = r2_score(y_test.values[:,0], y_pred[:,0])
    LR_test_Rsquared_2 = r2_score(y_test.values[:,1], y_pred[:,1])
    return [LR_test_mse_1,LR_test_mse_2,LR_test_mae_1,LR_test_mae_2,LR_test_rmse_1,LR_test_rmse_2, LR_test_Rsquared_1,LR_test_Rsquared_2]


# Helper for plotting learning curves
from sklearn.model_selection import learning_curve
import matplotlib.pyplot as plt

def plot_learning_curve(model, X, y, scoring='neg_mean_squared_error'):
    train_sizes, train_scores, test_scores = learning_curve(
        model, X, y, scoring=scoring, cv=5, train_sizes=np.linspace(0.1, 1.0, 5)
    )

    train_scores_mean = -train_scores.mean(axis=1)
    test_scores_mean = -test_scores.mean(axis=1)

    plt.figure()
    plt.plot(train_sizes, train_scores_mean, label='Training error')
    plt.plot(train_sizes, test_scores_mean, label='Cross-validation error')
    plt.xlabel('Training Set Size')
    plt.ylabel('Mean Squared Error')
    plt.title(f'Learning Curve - {model.__class__.__name__}')
    plt.legend()
    plt.grid(True)

    os.makedirs("learning_curves", exist_ok=True)
    filename = f"learning_curves/{model.__class__.__name__}_learning_curve.png"
    plt.savefig(filename)
    plt.close()