CODEWITHSUNDEEP
neural-networkpytorch
B_Miner
B_Miner

Reputation: 1820

Pytorch Forward Pass Changes Each Time?

I am learning pytorch and running a toy regression problem. I am baffled by the fact that it appears that each time I run a tensor through a model, the prediction changes. Clearly this is cant be the case but what am I missing?

Pytorch version: 0.4.0

I am running here without GPU to eliminate that potential issue.

Code:

from __future__ import division
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import torch
import torch.utils.data as utils_data
from torch.autograd import Variable
from torch import optim, nn
from torch.utils.data import Dataset 
import torch.nn.functional as F
from torch.nn.init import xavier_uniform_, xavier_normal_,uniform_
from sklearn.datasets import load_boston
from sklearn.metrics import mean_squared_error

cuda=False #set to true uses GPU


#load boston data from scikit
boston = load_boston()
x=boston.data
y=boston.target
y=y.reshape(y.shape[0],1)

#change to tensors
x = torch.from_numpy(x)
y = torch.from_numpy(y)

#create dataset and use data loader
training_samples = utils_data.TensorDataset(x, y)
data_loader = utils_data.DataLoader(training_samples, batch_size=64)

#simple model
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()

        #all the layers
        self.fc1   = nn.Linear(x.shape[1], 20)
        xavier_uniform_(self.fc1.weight.data) #this is how you can change the weight init
        self.drop = nn.Dropout(p=0.5)
        self.fc2   = nn.Linear(20, 1)


    def forward(self, x):
        x = F.relu(self.fc1(x))
        x=  self.drop(x)
        x = self.fc2(x)
        return x

net=Net()

if cuda:
    net.cuda()

# create a stochastic gradient descent optimizer
optimizer = optim.Adam(net.parameters())
# create a loss function (mse)
loss = nn.MSELoss(size_average=True)

# run the main training loop
epochs =50
hold_loss=[]

for epoch in range(epochs):
    cum_loss=0.
    for batch_idx, (data, target) in enumerate(data_loader):
        tr_x, tr_y = data.float(), target.float()
        if cuda:
            tr_x, tr_y = tr_x.cuda(), tr_y.cuda() 

        # Reset gradient
        optimizer.zero_grad()

        # Forward pass
        fx = net(tr_x)
        output = loss(fx, tr_y) #loss for this batch

        cum_loss += output.item() #accumulate the loss

        # Backward 
        output.backward()

        # Update parameters based on backprop
        optimizer.step()
    hold_loss.append(cum_loss/len(training_samples))  

#training loss
plt.plot(np.array(hold_loss))

This part, if re-ran will return different predictions each time, the actuals dont change so the order of the data is not changing!

#score the training set
for batch_idx, (data, target) in enumerate(data_loader):
        tr_x, tr_y = data.float(), target.float()
        if batch_idx ==0:
           hold_pred=net(tr_x).data.numpy()
           hold_actual=tr_y.data.numpy().reshape(tr_y.data.numpy().shape[0],1)
        else:
            hold_pred =np.row_stack([hold_pred,net(tr_x).data.numpy()])
hold_actual=np.row_stack([hold_actual,tr_y.data.numpy().reshape(tr_y.data.numpy().shape[0],1)])

#view the first few predictions      
print(hold_pred[0:10]) 
print(hold_actual[0:10])

Upvotes: 3

Views: 2449

Answers (1)

benjaminplanche
benjaminplanche

Reputation: 15119

Your network has a Dropout layer, which has for purpose to randomly sample (with a probability p=0.5 here) the data it receives during training (net.train() set before inference). See the doc for more information (usage, purpose).

This layer can be short-circuited during testing (net.eval() set before inference).

Upvotes: 8

Related Questions