How to replace Softmax ouput Layer with Logistic Layer in Tensorflow?

Question

I need a bit help in my work. Right now, I am using Softmax layer as output layer for classification scores in neural network. But, I need to replace Softmax layer with logistic layer on the output layer. I have some inputs that belongs to multiple classes. Softmax is showing probability over all the classes and assigned the class to the highest probability and its hard to decide a threshold to predict more than one classes at a time. While in case of logistic function each neuron will display a number between (0-1) and I can decide a threshold in that case. Here is my code:

2 layer Network Initialization

# Parameters
training_epochs = 10#100
batch_size = 64
display_step = 1
batch = tf.Variable(0, trainable=False)
regualarization =  0.009

# Network Parameters
n_hidden_1 = 250 # 1st layer num features
n_hidden_2 = 250 # 2nd layer num features

n_input = model.layer1_size # Vector input (sentence shape: 30*10) 
n_classes = 12 # Sentence Category detection total classes (0-11 categories)

#History storing variables for plots
loss_history = []
train_acc_history = []
val_acc_history = []


# tf Graph input
x = tf.placeholder("float", [None, n_input])
y = tf.placeholder("float", [None, n_classes])

#Strings
trainingString = "\n\nTraining Accuracy and Confusion Matrix:"
validationString = "\n\nValidation set Accuracy and Confusion Matrix:"
testString = "\n\nTest set Accuracy and Confusion Matrix:"
goldString = "\n\nGold set Accuracy and Confusion Matrix:"

# Create model
def multilayer_perceptron(_X, _weights, _biases):
    #Single Layer
    #layer_1 = tf.nn.relu(tf.add(tf.matmul(_X, _weights['h1']), _biases['b1'])) 
    #return tf.matmul(layer_1, weights['out']) + biases['out']

    ##2 layer
    #Hidden layer with RELU activation
    layer_1 = tf.nn.relu(tf.add(tf.matmul(_X, _weights['h1']), _biases['b1'])) 
    #Hidden layer with RELU activation
    layer_2 = tf.nn.relu(tf.add(tf.matmul(layer_1, _weights['h2']), _biases['b2'])) 
    return tf.matmul(layer_2, weights['out']) + biases['out']  

# Store layers weight & bias
weights = {
    ##1 Layer
    #'h1': w2v_utils.weight_variable(n_input, n_hidden_1),
    #'out': w2v_utils.weight_variable(n_hidden_1, n_classes)

    ##2 Layer
    'h1':  w2v_utils.weight_variable(n_input, n_hidden_1),
    'h2':  w2v_utils.weight_variable(n_hidden_1, n_hidden_2),
    'out': w2v_utils.weight_variable(n_hidden_2, n_classes)   
}

biases = {
    ##1 Layer
    #'b1': w2v_utils.bias_variable([n_hidden_1]),
    #'out': w2v_utils.bias_variable([n_classes])  

    ##2 Layer
    'b1': w2v_utils.bias_variable([n_hidden_1]),
    'b2': w2v_utils.bias_variable([n_hidden_2]),
    'out': w2v_utils.bias_variable([n_classes])
}

# Construct model
pred = multilayer_perceptron(x, weights, biases)

# Define loss and optimizer
#learning rate
# Optimizer: set up a variable that's incremented once per batch and
# controls the learning rate decay.
learning_rate = tf.train.exponential_decay(
    0.02*0.01,                # Base learning rate.
    batch * batch_size,  # Current index into the dataset.
    X_train.shape[0],    # Decay step.
    0.96,                # Decay rate.
    staircase=True)

#L2 regularization
l2_loss = tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables()])

#Softmax loss
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y)) 

#Total_cost
cost = cost+ (regualarization*0.5*l2_loss)

# Adam Optimizer
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost,global_step=batch)

 # Initializing the variables
 init = tf.initialize_all_variables()

 print "Network Initialized!"

How we can modify this network to have a probability in between (0-1) on each output Neuron?

Answer 1

just change line:

# Construct model
pred = multilayer_perceptron(x, weights, biases)

To

# Construct model
model pred = tf.nn.sigmoid(multilayer_perceptron(x, weights, biases))