Innovations Anonymous
Innovations Anonymous

Reputation: 134

Sympy Broadcasting Matrix-Vector Addition for MLP

I'm trying to get a grasp on declarative programming, so I've started learning Sympy and my "hello world" is to try to represent a standard MLP by converting a vanilla numpy implementation I found online. I am getting stuck trying to add the bias vector. Is there a differentiable way to do this operation in Sympy?

#! /usr/bin/env python3

import numpy as np
import random
import sympy as sp

i   = 3
o   = 1
x   = sp.Symbol('x')
w   = sp.Symbol('w')
b   = sp.Symbol('b')
y   = sp.Symbol('y')
Φ   = sp.tanh # activation function
mlp = Φ(x*w+b)
L   = lambda a, e: a - e # loss function
C   = L(mlp, y)
dC  = sp.diff(C, w) # partial deriv of C with respect to each weight
η   = 0.01               # learning rate
if __name__ == "__main__":
    random.seed(1)
    train_inputs  = np.array([[0, 0, 1], [1, 1, 1], [1, 0, 1], [0, 1, 1]])
    train_outputs = np.array([[0, 1, 1, 0]]).T
    W  = 2 * np.random.rand(i, o) - 1 # TODO parameterize initialization
    W  = sp.Matrix(W)
    B  = 2 * np.random.rand(1, o) - 1 # TODO parameterize initialization
    B  = sp.Matrix(B)
    for temp, ye in zip(train_inputs, train_outputs):
        X  = sp.Matrix(temp)
        ya = mlp.subs({'x':X, 'w':W, 'b':B}).n()
        Δ  = dC.subs({'y':ye, 'x':X, 'b':B, 'w':W}).n()
        W -= η * Δ
        b -= η * Δ

Upvotes: 0

Views: 102

Answers (0)

Related Questions