Neural Network and plot_decision_regions?

Question

I have created a neural network following the steps on this website:

https://machinelearningmastery.com/implement-backpropagation-algorithm-scratch-python/

And my neural network is this one:

from random import seed
from random import randrange
from random import random
from csv import reader
from math import exp


# Attivazione del neurone
def activate(pesi, inputs):
    activation = pesi[-1]
    for i in range(len(pesi)-1):
        activation += pesi[i] * inputs[i]
    return activation
 
# Transfer function scelta
def transfer(activation):
    return 1.0 / (1.0 + exp(-activation))
 
# Propagazione forward
def forward_propagate(network, row):
    inputs = row
    for layer in network:
        nuovi = []
        for neuron in layer:
            activation = activate(neuron['pesi'], inputs)
            neuron['output'] = transfer(activation)
            nuovi.append(neuron['output'])
        inputs = nuovi
    return inputs
 
# Derivata
def transfer_derivative(output):
    return output * (1.0 - output)
 
# Back Propagation
def backward_propagate_error(network, valore):
    for i in reversed(range(len(network))):
        layer = network[i]
        errorF = list()
        if i != len(network)-1:
            for j in range(len(layer)):
                errore = 0.0
                for neuron in network[i + 1]:
                    errore += (neuron['pesi'][j] * neuron['delta'])
                errorF.append(errore)
        else:
            for j in range(len(layer)):
                neuron = layer[j]
                errorF.append(valore[j] - neuron['output'])
        for j in range(len(layer)):
            neuron = layer[j]
            neuron['delta'] = errorF[j] * transfer_derivative(neuron['output'])

#aggiornamento pesi
def update_weights(network, row, Coeff):
    for i in range(len(network)):
        inputs = row[:-1]
        if i != 0:
            inputs = [neuron['output'] for neuron in network[i - 1]]
        for neuron in network[i]:
            for j in range(len(inputs)):
                neuron['pesi'][j] += Coeff * neuron['delta'] * inputs[j]
            neuron['pesi'][-1] += Coeff * neuron['delta']

#training della rete
def train_network(network, train, Coeff, NumEpoc, n_outputs):
    for epoch in range(NumEpoc):
        for row in train:
            outputs = forward_propagate(network, row)
            expected = [0 for i in range(n_outputs)]
            expected[int(row[-1])] = 1
            backward_propagate_error(network, expected)
            update_weights(network, row, Coeff)

#inizializzazione
def initialize_network(n_inputs, n_hidden, n_outputs):
    network = list()
    hidden_layer = [{'pesi':[random() for i in range(n_inputs + 1)]} for i in range(n_hidden)]
    network.append(hidden_layer)
    output_layer = [{'pesi':[random() for i in range(n_hidden + 1)]} for i in range(n_outputs)]
    network.append(output_layer)
    return network
 
# Predict
def predict( row):
    outputs = forward_propagate(network, row)
    return outputs.index(max(outputs))

It goes really well, I train him with:

seed(1)


dataset = np.insert(XL,2,yL,axis=1)
n_inputs = len(dataset[0]) - 1

n_outputs = len(set([int(row[-1]) for row in dataset]))
network = initialize_network(n_inputs, 3, n_outputs)
train_network(network, dataset, 0.2, 2000, n_outputs)

And I can predict with

x=predict(network, XT[i])

My dataset is this one:

and I create it with:

%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from mlxtend.plotting import plot_decision_regions
import matplotlib
matplotlib.rcParams['figure.figsize'] = [10, 10]

# data
def donut_data(N,noise=0.8):
    X = np.random.randn(N, 2)
    X_r = X + np.random.uniform(-noise,noise,X.shape)
    Y = np.logical_or(X_r[:, 0]*X_r[:, 0] + X_r[:, 1]*X_r[:, 1] < 0.3, X_r[:, 0]*X_r[:, 0] + X_r[:, 1]*X_r[:, 1]> 2)
    Y = np.where(Y, 1, -1)
    return X, Y

def plot_data(X,Y,c1='b',c2='r', toplot=True):
    plt.scatter(X[Y==1, 0], X[Y==1, 1],c=c1, marker='x', label='1')
    plt.scatter(X[Y==-1, 0], X[Y==-1, 1],c=c2, marker='s', label='-1')
    if toplot:
        plt.ylim(-3.0)
        plt.legend()
        plt.show()


N = 1000
noise = 0.5
XL, yL = donut_data(N, noise)
XT, yT = donut_data(N, noise)
plot_data(XL,yL,'b','r',True)
plot_data(XT,yT,'k','m', True)

I would like to use plot_decision_regions(XT, yT, network), to plot the regions but I have a problem with this program.

Network is a simple list not an object of a class like it is expected in plot_decision_regions. How can I overcome this? Have I to rewrite the programme, with network as an object of a class?

Answer 1

I think the plot_decision_regions() really needs a network (or any classifier) as object. This because inside the code it applies the .predict() method to the whole meshgrid, in order to know how the neural network classifies it.

Look at a similar function's code:

def plot_decision_regions(network, X, y, points=200):
    markers = ('o', '^')
    colors = ('red', 'blue')
    cmap = ListedColormap(colors)

    x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max() + 1

    resolution = max(x1_max - x1_min, x2_max - x2_min) / float(points)

    xx1, xx2 = numpy.meshgrid(numpy.arange(x1_min,
                                           x1_max,
                                           resolution),
                              numpy.arange(x2_min, x2_max, resolution))
    input = numpy.array([xx1.ravel(), xx2.ravel()]).T
    Z = numpy.empty(0)

  #### In this loop, it applies .predict() #######
    for i in range(input.shape[0]):
        val = network.predict(numpy.array(input[i]))
        if val < 0.5:
            val = 0
        if val >= 0.5:
            val = 1
        Z = numpy.append(Z, val)

    Z = Z.reshape(xx1.shape)

    plt.pcolormesh(xx1, xx2, Z, cmap=cmap)
    plt.xlim(xx1.min(), xx1.max())
    plt.ylim(xx2.min(), xx2.max())
    # plot all samples

    classes = ["False", "True"]

    for idx, cl in enumerate(numpy.unique(y)):
        plt.scatter(x=X[y == cl, 0],
                    y=X[y == cl, 1],
                    alpha=1.0,
                    c=colors[idx],
                    edgecolors='black',
                    marker=markers[idx],
                    s=80,
                    label=classes[idx])

    plt.xlabel('x-axis')
    plt.ylabel('y-axis')
    plt.legend(loc='upper left')
    plt.show()