XOR classification using multilayer perceptrons is outputting 1 for all inputs

Question

I'm using a neural network with 1 hidden layer (2 neurons) and 1 output neuron for solving the XOR problem.

Here's the code I'm using. It contains the main run file xor.py which creates a model defined in model.py. Each neuron is defined by the class Neuron in neuron.py

xor.py

from model import Model
import numpy as np

inputs = [[0,0], [0,1], [1,0], [1,1]]
outputs = [0, 1, 1, 0]

m = Model()

m.train(inputs, outputs)

for i in inputs:
    p = m.predict(i)
    print str(i) + ' => ' + str(p)

model.py

from neuron import HiddenNeuron, OutputNeuron
import numpy as np

    class Model(object):

    def __init__(self):
        self.hidden = [HiddenNeuron(2) for i in range(2)]
        self.output = OutputNeuron(2)

    def predict(self, input):
        temp = []
        for x in range(2):
            self.hidden[x].forward(input)
            temp.append(self.hidden[x].out)
        self.output.forward(temp)
        return self.output.out

    def train(self, inputs, targets):
        it = 0
        i = 0
        size = len(inputs)
        while it < 4:
            if i == size:
                i = 0
            feature = inputs[i]
            print '\n\nFeature : ' + str(feature) + '\n'
            print 'Output weights : ' + str(self.output.weights)
            print 'Hidden 1 weights : ' + str(self.hidden[0].weights)
            print 'Hidden 2 weights : ' + str(self.hidden[1].weights)
            temp = []
            for x in range(2):
                self.hidden[x].forward(feature)
                temp.append(self.hidden[x].out)
            self.output.forward(temp)
            self.output.backward(targets[i])
            deltas = []
            deltas.append(self.output.error)
            weights = []
            weights.append([self.output.weights[0]])
            weights.append([self.output.weights[1]])
            for x in range(2):
                self.hidden[x].backward(deltas, weights[x])
            for x in range(2):
                self.hidden[x].update(feature)
            self.output.update(temp)
            it += 1
            i += 1

neuron.py

import numpy as np
from random import uniform

class Neuron(object):

    def activation(self, fx):
        return 1/(1 + np.exp(-fx))

    def __init__(self, dim, lrate):
        self.dim = dim
        self.weights = np.empty([dim])
        self.weights = [uniform(0,1) for x in range(dim)]
        self.bias = uniform(0, 1)
        self.lrate = lrate
        self.out = None
        self.error = None

    def update(self, input):
        j = 0
        for i in input:
            delta = self.lrate * self.error
            self.weights[j] -= (delta*i)
            self.bias += delta
            j+=1

    def forward(self, input):
        j = 0
        sum = self.bias
        for f in input:
            sum += f * self.weights[j]
            j+=1
        self.out = self.activation(sum)

    def backward(self):
        pass

class OutputNeuron(Neuron):

    def __init__(self, dim, lrate=0.2):
        super(OutputNeuron, self).__init__(dim, lrate)

    def backward(self, target):
        self.error = self.out * (1 - self.out) * (self.out - target)


class HiddenNeuron(Neuron):

    def __init__(self, dim, lrate=0.2):
        super(HiddenNeuron, self).__init__(dim, lrate)

    def backward(self, deltas, weights):
        sum = 0
        size = len(deltas)
        for x in range(size):
            sum += deltas[x] * weights[x]
        self.error = self.out * (1 - self.out) * sum

The final output is

[0, 0] => 0.999999991272
[0, 1] => 0.999999970788
[1, 0] => 0.999999952345
[1, 1] => 0.999715564446

Answer 1

I think the error is in neuron.py in the function update(). If you change self.bias += delta to self.bias -= delta it should work, at least it does for me. Otherwise you would modify your biases to ascend towards a maximum on the error surface.

Below you can see the output after 100000 training epochs.

[0, 0] => 0.0174550173543 [0, 1] => 0.983899954593 [1, 0] => 0.983895388655 [1, 1] => 0.0164172288168