kora_num14
kora_num14

Reputation: 199

How to create a precision-recall curve plot to compare 2 classifiers in Python?

My goal is to find the precision-recall curve, comparing with Logistic Regression and Random Forest and plotting them in one graph. I wanted to know if I used the right steps to create a plot to compare both classifiers. I appreciate all the help!

Code:

from sklearn.preprocessing import MultiLabelBinarizer as mlb
import numpy as np
import pandas as pd
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.datasets import make_classification
from sklearn import metrics
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import f1_score
from sklearn.metrics import auc
from matplotlib import pyplot

X = df[["DIAGNOSIS_CD_Dummy"]]
y = df[["TEST_RESULT_Dummy"]]
# X = pd.DataFrame(df.iloc[:, -1])
# y = pd.DataFrame(df.iloc[:, :-1])

# raw confusion matrix
df = pd.DataFrame(df, columns=["DIAGNOSIS_CD_Dummy", "TEST_RESULT_Dummy"])
confusion_matrix = pd.crosstab(
    df["TEST_RESULT_Dummy"],
    df["DIAGNOSIS_CD_Dummy"],
    rownames=["Test Result"],
    colnames=["Diagnosis"],
)
print(confusion_matrix)


# Logistic Regression Confusion Matrix
from sklearn.preprocessing import MultiLabelBinarizer as mlb
import numpy as np
import pandas as pd
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.datasets import make_classification
from sklearn import metrics


# split into training and test using scikit
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    X, y.values.ravel(), test_size=0.3, random_state=1, stratify=y
)
log_model = LogisticRegression()
log_model.fit(X_train, y_train)

# use logistic regression model to make predictions
y_score = log_model.predict_proba(X_test)[:, 1]

y_pred = log_model.predict(X_test)
y_pred = np.round(y_pred)
confusion_matrix = confusion_matrix(y_test, y_pred)
print("\n")
print(confusion_matrix)
print("\n")
print(classification_report(y_test, y_pred, zero_division=0))

# calculate precision and recall
precision, recall, thresholds = precision_recall_curve(y_test, y_score)

# create precision recall curve
fig, ax = plt.subplots()
ax.plot(recall, precision, color="purple")

# add axis labels to plot
ax.set_title("Precision-Recall Curve")
ax.set_ylabel("Precision")
ax.set_xlabel("Recall")

# display plot
plt.show()

# precision-recall curve
# generate 2 class dataset
X = df[["DIAGNOSIS_CD_Dummy"]]
y = df[["TEST_RESULT_Dummy"]]

# X = pd.DataFrame(df.iloc[:, :-1])
# y = pd.DataFrame(df.iloc[:, -1])

# split into train/test sets
trainX, testX, trainy, testy = train_test_split(
    X, y.values.ravel(), test_size=0.3, random_state=2
)
# fit a model
model = LogisticRegression(solver="lbfgs")
model.fit(trainX, trainy)

# predict probabilities
lr_probs = model.predict_proba(testX)
# probs_rf = model_rf.predict_proba(testX)[:, 1]

# keep probabilities for the positive outcome only
lr_probs = lr_probs[:, 1]

# predict class values
yhat = model.predict(testX)
lr_precision, lr_recall, _ = precision_recall_curve(testy, lr_probs)
lr_f1, lr_auc = f1_score(testy, yhat), auc(lr_recall, lr_precision)

# precision_rf, recall_rf, _ = precision_recall_curve(testy, probs_rf)
# f1_rf, auc_rf = f1_score(testy, yhat), auc(recall_rf, precision_rf)
# auc_rf = auc(recall_rf, precision_rf)


# summarize scores
print("Logistic: f1=%.3f auc=%.3f" % (lr_f1, lr_auc))

# plot the precision-recall curves
no_skill = len(testy[testy == 1]) / len(testy)
pyplot.plot([0, 1], [no_skill, no_skill], linestyle="--", label="No Skill")
pyplot.plot(lr_recall, lr_precision, marker=".", label="Logistic")

plt.plot(lr_precision, lr_recall, label=f"AUC (Logistic Regression) = {lr_auc:.2f}")

# axis labels
pyplot.xlabel("Recall")
pyplot.ylabel("Precision")
# show the legend
pyplot.legend()
# show the plot
pyplot.show()



# Random Forest
model_rf = RandomForestClassifier()
model_rf.fit(trainX, trainy)
# model_rf = RandomForestClassifier().fit(trainX, trainy)

# predict probabilities
lr_probs = model.predict_proba(testX)
probs_rf = model_rf.predict_proba(testX)

# keep probabilities for the positive outcome only
probs_rf = probs_rf[:, 1]

# predict class values
yhat = model.predict(testX)
precision_rf, recall_rf, _ = precision_recall_curve(testy, probs_rf)
f1_rf, auc_rf = f1_score(testy, yhat), auc(recall_rf, precision_rf)
auc_rf = auc(recall_rf, precision_rf)

print("Random Forest: f1=%.3f auc=%.3f" % (f1_rf, auc_rf))

# plot the precision-recall curves
no_skill = len(testy[testy == 1]) / len(testy)
pyplot.plot([0, 1], [no_skill, no_skill], linestyle="--", label="No Skill")
pyplot.plot(lr_recall, lr_precision, marker=".", label="Random Forest")

plt.plot(recall_rf, precision_rf, label=f"AUC (Random Forests) = {auc_rf:.2f}")

# axis labels
pyplot.xlabel("Recall")
pyplot.ylabel("Precision")
# show the legend
pyplot.legend()
# show the plot
pyplot.show()

Output:

Diagnosis        0    1
Test Result            
0            18385   32
1             1268  165


[[5514   11]
 [ 374   56]]


              precision    recall  f1-score   support

           0       0.94      1.00      0.97      5525
           1       0.84      0.13      0.23       430

    accuracy                           0.94      5955
   macro avg       0.89      0.56      0.60      5955
weighted avg       0.93      0.94      0.91      5955

Logistic: f1=0.193 auc=0.488

Logistic

Random Forest: f1=0.193 auc=0.488

Random Forest

Upvotes: 2

Views: 1736

Answers (1)

Ozer Ozdal
Ozer Ozdal

Reputation: 86

This is my attempt to plot it.

import pathlib
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.ticker import (MultipleLocator, AutoMinorLocator)
from sklearn.metrics import PrecisionRecallDisplay
from sklearn.multiclass import OneVsRestClassifier
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import average_precision_score
from itertools import cycle
from imblearn.pipeline import Pipeline
from sklearn.preprocessing import label_binarize

def __plot_binary_precision_recall_curve(X_test, y_test, *args, **kwargs):
    """
    Private function to be used by plot_precision_recall_curve for binary applications.
    """

    if 'fig_size' in kwargs and 'dpi' in kwargs:
        fig, ax = plt.subplots(figsize=kwargs['fig_size'], dpi=kwargs['dpi'])
    else:
        fig, ax = plt.subplots()

    plt.rcParams["figure.facecolor"] = 'white'
    plt.rcParams["axes.facecolor"] = 'white'
    plt.rcParams["savefig.facecolor"] = 'white'

    ax.xaxis.set_major_locator(MultipleLocator(0.1))
    ax.xaxis.set_major_formatter('{x:.1f}')

    ax.yaxis.set_major_locator(MultipleLocator(0.1))
    ax.yaxis.set_major_formatter('{x:.1f}')

    ax.xaxis.set_minor_locator(MultipleLocator(0.05))
    ax.yaxis.set_minor_locator(MultipleLocator(0.05))

    ax.tick_params(which='both', width=2)
    ax.tick_params(which='major', length=7)
    ax.tick_params(which='minor', length=4, color='black')

    plt.grid(True, zorder=0)
    plt.plot([0, 1], [1, 0], linestyle='--', lw=1, color='k',
             label='Luck', alpha=.8, zorder=1) # random prediction curve
    plt.plot([1, 1], [1, 0], c='k', linestyle='dashdot'), plt.plot([1, 1], c='k', linestyle='dashdot', zorder=2, label="Perfect model") #perfect model prediction curve

    f_scores = np.linspace(0.2, 0.8, num=4)
    lines, labels = [], []
    for f_score in f_scores:
        x = np.linspace(0.01, 1)
        y = f_score * x / (2 * x - f_score)
        (l,) = plt.plot(x[y >= 0], y[y >= 0], color="gray", alpha=0.2)
        plt.annotate("f1={0:0.1f}".format(f_score), xy=(0.9, y[45] + 0.02))

    zorder = 3
    for classifier in args:
        display = PrecisionRecallDisplay.from_estimator(classifier, X_test, y_test, ax=ax, zorder=zorder)
        zorder +=1

    # add the legend for the iso-f1 curves
    handles, labels = display.ax_.get_legend_handles_labels()
    handles.extend([l])
    labels.extend(["iso-f1 curves"])
    # set the legend and the axes
    ax.set_xlim([0.0, 1.0])
    ax.set_ylim([0.0, 1.05])
    ax.legend(handles=handles, labels=labels, loc="best")

    plt.xlabel('Recall', fontsize=18)
    plt.ylabel('Precision', fontsize=18)

    if 'title' in kwargs:
        ax.set_title(kwargs['title'], fontsize=18)
    else:
        ax.set_title("Precision-Recall Curve", fontsize=18)

    if 'save_fig_path' in kwargs:
        path = pathlib.Path(kwargs['save_fig_path'])
        path.parent.mkdir(parents=True, exist_ok=True)
        fig.savefig(kwargs['save_fig_path'], dpi=kwargs['dpi'], facecolor=fig.get_facecolor(), edgecolor='none')

    return fig, ax

def __plot_multiclass_precision_recall_curve(X_train, y_train, X_test, y_test, *args, **kwargs):
    """
    Private function designed to be used by plot_precision_recall_curve for multiclass applications.
    """

    my_vals = y_test.unique().tolist()
    my_vals.sort()

    # binarize the y_test series
    y_test = label_binarize(y_test, classes=my_vals)

    n_classes = y_test.shape[1]

    # setup plot details
    colors = cycle(["navy", "turquoise", "darkorange", "cornflowerblue", "teal"])

    if 'fig_size' in kwargs and 'dpi' in kwargs:
        fig, ax = plt.subplots(len(args), figsize=kwargs['fig_size'], dpi=kwargs['dpi'], facecolor='white')
    else:
        fig, ax = plt.subplots(len(args), facecolor='white')

    for count, clfs in enumerate(args):

        ax[count].xaxis.set_major_locator(MultipleLocator(0.1))
        ax[count].xaxis.set_major_formatter('{x:.1f}')

        ax[count].yaxis.set_major_locator(MultipleLocator(0.1))
        ax[count].yaxis.set_major_formatter('{x:.1f}')

        ax[count].xaxis.set_minor_locator(MultipleLocator(0.05))
        ax[count].yaxis.set_minor_locator(MultipleLocator(0.05))

        ax[count].tick_params(which='both', width=2)
        ax[count].tick_params(which='major', length=7)
        ax[count].tick_params(which='minor', length=4, color='black')

        ax[count].grid(True, zorder=0)
        ax[count].plot([0, 1], [1, 0], linestyle='--', lw=1, color='k',
             label='Luck', alpha=.8, zorder=1) # random prediction curve
        ax[count].plot([1, 1], [1, 0], c='k', linestyle='dashdot'), ax[count].plot([1, 1], c='k', linestyle='dashdot', zorder=2, label="Perfect model") #perfect model prediction curve

        # set up the model, wrapped by the OneVsRestClassifier
        classifier = OneVsRestClassifier(clfs)
        classifier.fit(X_train, y_train) # train the model

        # produce the predictions (as probabilities)
        y_score = classifier.predict_proba(X_test)

        # For each class
        precision = dict()
        recall = dict()
        average_precision = dict()
        for i in range(n_classes):
            precision[i], recall[i], _ = precision_recall_curve(y_test[:, i], y_score[:, i])
            average_precision[i] = average_precision_score(y_test[:, i], y_score[:, i])

        # A "micro-average": quantifying score on all classes jointly
        precision["micro"], recall["micro"], _ = precision_recall_curve(
            y_test.ravel(), y_score.ravel()
        )
        average_precision["micro"] = average_precision_score(y_test, y_score, average="micro")

        f_scores = np.linspace(0.2, 0.8, num=4)
        lines, labels = [], []
        for f_score in f_scores:
            x = np.linspace(0.01, 1)
            y = f_score * x / (2 * x - f_score)
            (l,) = ax[count].plot(x[y >= 0], y[y >= 0], color="gray", alpha=0.2)
            ax[count].annotate("f1={0:0.1f}".format(f_score), xy=(0.9, y[45] + 0.02))

        display = PrecisionRecallDisplay(
            recall=recall["micro"],
            precision=precision["micro"],
            average_precision=average_precision["micro"],
            )
        display.plot(ax=ax[count], name="Micro-average precision-recall", color="gold")

        for i, color in zip(range(n_classes), colors):
            display = PrecisionRecallDisplay(
                recall=recall[i],
                precision=precision[i],
                average_precision=average_precision[i],
            )
            display.plot(ax=ax[count], name=f"Precision-recall for class {i}", color=color)

        # add the legend for the iso-f1 curves
        handles, labels = display.ax_.get_legend_handles_labels()
        handles.extend([l])
        labels.extend(["iso-f1 curves"])
        # set the legend and the axes
        ax[count].set_xlim([0.0, 1.0])
        ax[count].set_ylim([0.0, 1.05])
        ax[count].legend(handles=handles, labels=labels, loc="best")

        if type(clfs) == Pipeline:
            estimator_name = str(type(clfs['clf'])).split(".")[-1][:-2]
        else:
            estimator_name = str(type(clfs)).split(".")[-1][:-2]

        if 'title' in kwargs:
            ax[count].set_title(kwargs['title'] + " - " + estimator_name, fontsize=18)
        else:
            ax[count].set_title("Precision-Recall Curve" + " - " + estimator_name, fontsize=18)

        ax[count].set_xlabel('Recall', fontsize=18)
        ax[count].set_ylabel('Precision', fontsize=18)

        if 'save_fig_path' in kwargs:
            path = pathlib.Path(kwargs['save_fig_path'])
            path.parent.mkdir(parents=True, exist_ok=True)
            fig.savefig(kwargs['save_fig_path'], dpi=kwargs['dpi'], facecolor=fig.get_facecolor(), edgecolor='none')

    return fig, ax


def plot_precision_recall_curve(X_train, y_train, X_test, y_test, *args, **kwargs):
    """
    Plots precision recall curves for the given models

    Parameters
    ----------
    X_test : pandas.DataFrame of shape (n_samples, n_features)
        Test values.
    y_test : pandas.Series of shape (n_samples,)
        Target values.
    *args : estimators to plot precision and recall curves
        estimator instance (either sklearn.Pipeline, imblearn.Pipeline or a classifier)
        PRE-FITTED classifier or a PRE-FITTED Pipeline in which the last estimator is a classifier.
    **kwargs : The following options are available with kwargs
        fig_size : tuple
            Size (inches) of the plot.
        dpi : int, default = 100
            Image DPI.
        title : str
            The title of the plot.
        save_fig_path : str
            Full path where to save the plot. Will generate the folders if they don't exist already.

    Returns
    -------
        fig : Matplotlib.pyplot.Figure
            Figure from matplotlib
        ax : Matplotlib.pyplot.Axe
            Axe object from matplotlib

    Example Syntax
    --------------
    fig, ax = reporting.plot_precision_recall_curve(X_train, y_train, X_test, y_test,
                            rf_pipe, catboost_classifier,
                            fig_size=(10,16), dpi=100,
                            title="Precision-Recall Curve",
                            save_fig_path="dir1/dir2/precision_recall_curve.png")

    """

    if (len(y_test.unique()) == 2):
        fig, ax = __plot_binary_precision_recall_curve(X_test, y_test, *args, **kwargs)
    else:
        fig, ax = __plot_multiclass_precision_recall_curve(X_train, y_train, X_test, y_test, *args, **kwargs)

    return fig, ax
  • Syntax and Output for Binary Classification
fig, ax = plot_precision_recall_curve(X_train, y_train, X_test, y_test,
               rf_pipe, xgboost_classifier,
               fig_size=(10,8), dpi=100,
               title="Precision-Recall Curve",
               save_fig_path="dir1/dir2/precision_recall_curve.png")

enter image description here

  • Syntax and Output for Multi-class Classification
fig, ax = plot_precision_recall_curve(X_train, y_train, X_test, y_test,
              rf_pipe, catboost_classifier,
              fig_size=(10,16), dpi=100,
              title="Precision-Recall Curve",
              save_fig_path="dir1/dir2/precision_recall_curve.png")

enter image description here

Upvotes: 2

Related Questions