RuntimeError: The shape of the mask [1682] at index 0 does not match the shape of the indexed tensor [1, 1682] at index 0

Question

I am designing an stacked autoencoder trying to train my neural network on movie rating if the user doesnt rate any movie it will not consider it

My training set runs perfectly but when i run test set it shows me this error

RuntimeError: The shape of the mask [1682] at index 0 does not match the shape of the indexed tensor [1, 1682] at index 0 I got error at the end test block i have commented there

CODE:-

---

# Auto Encoder


import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn as nn
import torch.nn.parallel
import torch.optim as optim
import torch.utils.data
from torch.autograd import Variable



# Importing dataset
movies= pd.read_csv('ml-1m/movies.dat',sep ='::', header= None,engine ='python', encoding= 'latin-1')
users= pd.read_csv('ml-1m/users.dat',sep ='::', header= None,engine ='python', encoding= 'latin-1')
ratings = pd.read_csv('ml-1m/ratings.dat',sep ='::', header= None,engine ='python', encoding= 'latin-1')

# preparing the training set and the dataset

training_set =pd.read_csv('ml-100k/u1.base',delimiter ='\t')
training_set =np.array(training_set, dtype= 'int')

test_set =pd.read_csv('ml-100k/u1.test',delimiter ='\t')
test_set =np.array(test_set, dtype= 'int')


# Getting the number of users and  movies
# we are taking the maximum no of values from training set and test set 

nb_users = int(max(max(training_set[:,0]), max(test_set[:,0])))
nb_movies = int(max(max(training_set[:,1]), max(test_set[:,1])))

# converting the data into an array within users in lines and movies in columns

def convert(data):
    new_data = []
    for id_users in range(1, nb_users +1):
        id_movies = data[:,1][data[:,0]==id_users]#movies id from data
        id_ratings = data[:,2][data[:,0]==id_users] #ratings
        ratings= np.zeros(nb_movies)
        ratings[id_movies-1] = id_ratings  # -1 for making it start from 1
        new_data.append(list(ratings))
    return new_data



training_set =convert(training_set)
test_set =convert(test_set)

# Converting the data into Torch tensor
training_set = torch.FloatTensor(training_set)
test_set = torch.FloatTensor(test_set)



# creating the architecture of the neural network
class SAE(nn.Module):

    def  __init__(self, ): # after comma it will consider parameters of module ie parent class
        super(SAE,self).__init__()#parent class inheritence
        self.fc1 = nn.Linear(nb_movies, 20)  #20 nodes in hidden layer
        self.fc2= nn.Linear(20,10)
        self.fc3 = nn.Linear(10,20)  #decoding 
        self.fc4= nn.Linear(20, nb_movies) #decoding
        self.activation= nn.Sigmoid()
            #self.myparameters= nn.ParameterList(self.fc1,self.fc2,self.fc3,self.fc4,self.activation)

    def forward(self, x): 
        x=self.activation(self.fc1(x))#encoding
        x=self.activation(self.fc2(x))#encoding
        x=self.activation(self.fc3(x)) #decoding
        x=self.fc4(x) #last layer machine understand automaically
        return x

sae= SAE()
criterion = nn.MSELoss()

optimizer= optim.RMSprop(sae.parameters(), lr= 0.01 , weight_decay =0.5)

# Training the SAE
nb_epoch = 200
for epoch in range(1, nb_epoch + 1):
    train_loss = 0
    s = 0.
    for id_user in range(nb_users):
        input = Variable(training_set[id_user]).unsqueeze(0)
        target = input.clone()
        if torch.sum(target.data > 0) > 0:
            output = sae(input)
            target.require_grad = False
            output[target == 0] = 0     
            loss = criterion(output, target)
            mean_corrector = nb_movies/float(torch.sum(target.data > 0) + 1e-10)
            loss.backward()
            train_loss += np.sqrt(loss.data.item()*mean_corrector)
            s += 1.
            optimizer.step()
    print('epoch: '+str(epoch)+' loss: '+str(train_loss/s))


# Testing the SAE
test_loss = 0
s = 0.
for id_user in range(nb_users):
    input = Variable(training_set[id_user]).unsqueeze(0)
    target = Variable(test_set[id_user])
    if torch.sum(target.data > 0) > 0:
        output = sae(input)
        target.require_grad = False
        output[target == 0] = 0      # I get error at this line
        loss = criterion(output, target)
        mean_corrector = nb_movies/float(torch.sum(target.data > 0) + 1e-10)
        test_loss += np.sqrt(loss.data.item()*mean_corrector)
        s += 1.
print('test loss: '+str(test_loss/s))

Answer 1

If you look in the training your SAE, the target is a clone of input which has an added dimension through the .unsqueeze(0) function.

If you look in the test your SAE, the target does not have the added dimension, therefore modify your code as follows

Change
target = Variable(test_set[id_user])

To
target = Variable(test_set[id_user]).unsqueeze(0)

So that it has that your target has the more than one dim the tensor requires.

Answer 2

Change:

output[target == 0] = 0      # I get error at this line

To:

output[(target == 0).unsqueeze(0)] = 0

Reason:

The torch.Tensor returned by target == 0 is of the shape [1682].

(target == 0).unsqueeze(0) will convert it to [1, 1682]