Neural network can learn |sin(x)| for [0,pi] but not [0,2pi] or [0, 4pi]

Question

My neural network can learn |sin(x)| for [0,pi], but not larger intervals than that. I tried changing the quantity and widths of hidden layers in various ways, but none of the changes leads to a good result.

I train the NN on thousands of random values from a uniform distribution in the chosen interval. using back propagation with gradient descent.

I am starting to think there is a fundamental problem in my network.

For the following examples I used a 1-10-10-1 layer structure:

[0, pi]:

[0, 2pi]:

[0, 4pi]:

Here is the code for the neural network:

import math
import numpy
import random
import copy
import matplotlib.pyplot as plt

def sigmoid(x):
    return 1.0/(1+ numpy.exp(-x))


def sigmoid_derivative(x):
    return x * (1.0 - x)


class NeuralNetwork:
    def __init__(self, weight_dimensions, x=None, y=None):
        self.weights = []
        self.layers = [[]] * len(weight_dimensions)
        self.weight_gradients = []
        self.learning_rate = 1

        self.layers[0] = x

        for i in range(len(weight_dimensions) - 1):
            self.weights.append(numpy.random.rand(weight_dimensions[i],weight_dimensions[i+1]) - 0.5)

        self.y = y

    def feed_forward(self):
        # calculate an output using feed forward layer-by-layer
        for i in range(len(self.layers) - 1):
            self.layers[i + 1] = sigmoid(numpy.dot(self.layers[i], self.weights[i]))

    def print_loss(self):
        loss = numpy.square(self.layers[-1] - self.y).sum()
        print(loss)


    def get_weight_gradients(self):
        return self.weight_gradients

    def apply_weight_gradients(self):
        for i in range(len(self.weight_gradients)):
            self.weights[i] += self.weight_gradients[i] * self.learning_rate
        if self.learning_rate > 0.001:
            self.learning_rate -= 0.0001

    def back_prop(self):
        # find derivative of the loss function with respect to weights
        self.weight_gradients = []
        deltas = []
        output_error = (self.y - self.layers[-1])
        output_delta = output_error * sigmoid_derivative(self.layers[-1])
        deltas.append(output_delta)
        self.weight_gradients.append(self.layers[-2].T.dot(output_delta))

        for i in range(len(self.weights) - 1):
            i_error = deltas[i].dot(self.weights[-(i+1)].T)
            i_delta = i_error * sigmoid_derivative(self.layers[-(i+2)])
            self.weight_gradients.append(self.layers[-(i+3)].T.dot(i_delta))
            deltas.append(copy.deepcopy(i_delta))

        # Unreverse weight gradient list
        self.weight_gradients = self.weight_gradients[::-1]

    def get_output(self, inp):
        self.layers[0] = inp
        self.feed_forward()
        return self.layers[-1]


def sin_test():
    interval = numpy.random.uniform(0, 2*math.pi, int(1000*(2*math.pi)))
    x_values = []
    y_values = []
    for i in range(len(interval)):
        y_values.append([abs(math.sin(interval[i]))])
        x_values.append([interval[i]])

    x = numpy.array(x_values)
    y = numpy.array(y_values)
    nn = NeuralNetwork([1, 10, 10, 1], x, y)

    for i in range(10000):
        tmp_input = []
        tmp_output = []
        mini_batch_indexes = random.sample(range(0, len(x)), 10)

        for j in mini_batch_indexes:
            tmp_input.append(x[j])
            tmp_output.append(y[j])
        nn.layers[0] = numpy.array(tmp_input)
        nn.y = numpy.array(tmp_output)
        nn.feed_forward()
        nn.back_prop()
        nn.apply_weight_gradients()
        nn.print_loss()
    nn.layers[0] = numpy.array(numpy.array(x))
    nn.y = numpy.array(numpy.array(y))
    nn.feed_forward()
    axis_1 = []
    axis_2 = []
    for i in range(len(nn.layers[-1])):
        axis_1.append(nn.layers[0][i][0])
        axis_2.append(nn.layers[-1][i][0])

    true_axis_2 = []
    for x in axis_1:
        true_axis_2.append(abs(math.sin(x)))

    axises = []
    for i in range(len(axis_1)):
        axises.append([axis_1[i], axis_2[i], true_axis_2[i]])

    axises.sort(key=lambda x: x[0], reverse=False)
    axis_1_new = []
    axis_2_new = []
    true_axis_2_new = []
    for elem in axises:
        axis_1_new.append(elem[0])
        axis_2_new.append(elem[1])
        true_axis_2_new.append(elem[2])
    plt.plot(axis_1_new, axis_2_new, label="nn")

    plt.plot(axis_1_new, true_axis_2_new, 'k--', label="sin(x)")
    plt.grid()
    plt.axis([0, 2*math.pi, -1, 2.5])
    plt.show()


sin_test()

Answer 1

The main issue with your network seem to be that you apply the activation function to the final "layer" of your network. The final output of your network should be a linear combination without any sigmoid applied.

As a warning though, do not expect the model to generalize outside of the region included in the training data.

Here is an example in PyTorch:

import torch
import torch.nn as nn
import math
import numpy as np

import matplotlib.pyplot as plt


N = 1000

p = 2.5
x = 2 * p * math.pi * torch.rand(N, 1)
y = np.abs(np.sin(x))

with torch.no_grad():
    plt.plot(x.numpy(), y.numpy(), '.')
    plt.savefig("training_data.png")

inner = 20

model = nn.Sequential(
        nn.Linear(1, inner, bias=True),
        nn.Sigmoid(),
        nn.Linear(inner, 1, bias=True)#,
        #nn.Sigmoid()
        )

loss_fn = nn.MSELoss()

learning_rate = 1e-3
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

for t in range(500000):
    y_pred = model(x)
    loss = loss_fn(y_pred, y)

    if t % 1000 == 0:
        print("MSE: {}".format(t), loss.item())

    model.zero_grad()

    loss.backward()

    optimizer.step()


with torch.no_grad():
    X = torch.arange(0, p * 2 * math.pi, step=0.01).reshape(-1, 1)
    Y = model(X)
    Y_TRUTH = np.abs(np.sin(X))
    print(Y.shape)
    print(Y_TRUTH.shape)
    loss = loss_fn(Y, Y_TRUTH)

    plt.clf()
    plt.plot(X.numpy(), Y_TRUTH.numpy())
    plt.plot(X.numpy(), Y.numpy())
    plt.title("MSE: {}".format(loss.item()))
    plt.savefig("output.png")

The output is available here: Image showing neural network prediction and ground truth. The yellow line is the predicted line by the neural network and the blue line is the ground truth.

Answer 2

First and foremost, you've chosen a topology suited for a different class of problems. A simple, fully-connected NN such as this is great with trivial classification (e.g. Boolean operators) or functions with at least two continuous derivatives. You've tried to apply it to a function that is simply one step beyond its capabilities.

Try your model on sin(x) and see how it performs at larger ranges. Try it on max(sin(x), 0). Do you see how the model has trouble with certain periodicity and irruptions? These are an emergent feature of the many linear equations struggling to predict the proper functional value: the linear combinations have trouble emulating non-linearities past a simple level.