CODEWITHSUNDEEP
pythonnumpymachine-learningneural-network
Atom
Atom

Reputation: 57

Network Cost Function code Python Implementation

I was implementing Andrew NG’s ML course in Python and in week 5 exercise 4 I was referring to a code. What I didn’t understand was the need to use np.trace() in the final output. Having a problem visualising the matrices

import numpy as np
from scipy.optimize import minimize
import scipy.io
import matplotlib.pyplot as plt

data_dict = scipy.io.loadmat('ex4_orig_octave/ex4data1.mat')

X = data_dict['X']
y = data_dict['y'].ravel()

M = X.shape[0]
N = X.shape[1] 
L = 26 # = number of nodes in the hidden layer (including bias node)
K = len(np.unique(y))

X = np.hstack((np.ones((M, 1)), X))

Y = np.zeros((M, K), dtype='uint8') 

for i, row in enumerate(Y):
    Y[i, y[i] - 1] = 1

weights_dict = scipy.io.loadmat('ex4_orig_octave/ex4weights.mat')

theta_1 = weights_dict['Theta1']
theta_2 = weights_dict['Theta2']

nn_params_saved = np.concatenate((theta_1.flatten(), theta_2.flatten()))

def nn_cost_function(nn_params, X, Y, M, N, L, K):
    """Python version of nnCostFunction.m after completing 'Part 1'."""

    # Unroll the parameter vector.
    theta_1 = nn_params[:(L - 1) * (N + 1)].reshape(L - 1, N + 1)
    theta_2 = nn_params[(L - 1) * (N + 1):].reshape(K, L)
    
    # Calculate activations in the second layer.
    a_2 = sigmoid(theta_1.dot(X.T))
    
    # Add the second layer's bias node.
    a_2_p = np.vstack((np.ones(M), a_2))
    
    # Calculate the activation of the third layer.
    a_3 = sigmoid(theta_2.dot(a_2_p))
    
    # Calculate the cost function.
    cost = 1 / M * np.trace(- Y.dot(np.log(a_3)) - (1 - Y).dot(np.log(1 - a_3)))
    
    return cost

cost_saved = nn_cost_function(nn_params_saved, X, Y, M, N, L, K) 

print 'Cost at parameters (loaded from ex4weights): %.6f' % cost_saved
print '(this value should be about 0.287629)'

Upvotes: 0

Views: 200

Answers (1)

Skrt
Skrt

Reputation: 119

The operation 1/M * np.trace() is calculating the average cost over a batch of size M:

enter image description here

A bit less readable, but significantly faster should be:

np.sum(np.sum(Y.multiply(np.log(a_3.T)),axis=1),axis=0)

, if Y.shape==(M,K) and a_3.shape==(K,M):

Y = lambda : np.random.uniform(size=(5000,10)) # (M,K)
a3 = lambda : np.random.uniform(size=(10,5000)) # (K,M)
timeit.timeit('import numpy as np; np.trace(Y().dot(a3()))', number=10, globals=globals())
# 0.5633535870001651
timeit.timeit('import numpy as np; np.sum(np.sum(np.multiply(Y(),a3().T),axis=1),axis=0)', number=10, globals=globals())
# 0.013223066000136896

Upvotes: 1

Related Questions