Reputation: 134
I'm trying to get a grasp on declarative programming, so I've started learning Sympy and my "hello world" is to try to represent a standard MLP by converting a vanilla numpy implementation I found online. I am getting stuck trying to add the bias vector. Is there a differentiable way to do this operation in Sympy?
#! /usr/bin/env python3
import numpy as np
import random
import sympy as sp
i = 3
o = 1
x = sp.Symbol('x')
w = sp.Symbol('w')
b = sp.Symbol('b')
y = sp.Symbol('y')
Φ = sp.tanh # activation function
mlp = Φ(x*w+b)
L = lambda a, e: a - e # loss function
C = L(mlp, y)
dC = sp.diff(C, w) # partial deriv of C with respect to each weight
η = 0.01 # learning rate
if __name__ == "__main__":
random.seed(1)
train_inputs = np.array([[0, 0, 1], [1, 1, 1], [1, 0, 1], [0, 1, 1]])
train_outputs = np.array([[0, 1, 1, 0]]).T
W = 2 * np.random.rand(i, o) - 1 # TODO parameterize initialization
W = sp.Matrix(W)
B = 2 * np.random.rand(1, o) - 1 # TODO parameterize initialization
B = sp.Matrix(B)
for temp, ye in zip(train_inputs, train_outputs):
X = sp.Matrix(temp)
ya = mlp.subs({'x':X, 'w':W, 'b':B}).n()
Δ = dC.subs({'y':ye, 'x':X, 'b':B, 'w':W}).n()
W -= η * Δ
b -= η * Δ
Upvotes: 0
Views: 102