How to add custom image sharpening layer in keras?

Question

I'm building a model in keras that detects building rooftops from images. I want to add a custom sharpening layer inside model. I know we can sharpen images in preprocessing, but it would be nice if i add a layer. I tried Lambda layer with my custom sharpening function but it didn't worked, then i tried a custom layer and got the same error:

TypeError                                 Traceback (most recent call last)
<ipython-input-5-3062d6b8160d> in <module>()
      1 # MY MODEL
      2 from u_net import mymodel300
----> 3 model = mymodel300((300, 300, 3))

~\Desktop\SAVERA\MYCODE\u_net.py in mymodel300(input_shape)
    186     # LAYERS
    187     inputs = Input(shape=input_shape)
--> 188     sharp = Sharpen(num_outputs=(300,300,3))(inputs)
    189     # 300x300
    190 

~\Anaconda3\lib\site-packages\tensorflow\python\keras\engine\base_layer.py in __call__(self, inputs, *args, **kwargs)
    701 
    702       if not in_deferred_mode:
--> 703         outputs = self.call(inputs, *args, **kwargs)
    704         if outputs is None:
    705           raise ValueError('A layer\'s `call` method should return a Tensor '

~\Desktop\SAVERA\MYCODE\u_net.py in call(self, input_)
    179     kernel_sharp = np.array(([-2, -2, -2], [-2, 17, -2], [-2, -2, -2]), dtype='int')
    180     #denoised = cv2.fastNlMeansDenoisingColored(img,None,5,5,2,10)
--> 181     self.sharp = cv2.filter2D(input_, -1, kernel_sharp)
    182     return self.sharp
    183 

TypeError: src is not a numpy array, neither a scalar

Here is my custom layer:

from tensorflow.keras.layers import Layer
from tensorflow.keras import backend as K
# CUSTOM SHARPEN LAYER
class Sharpen(Layer):
  def __init__(self, num_outputs):
    super(Sharpen, self).__init__()
    self.num_outputs = num_outputs

  def call(self, input_):
    import numpy as np
    import cv2
    # SHARPEN
    kernel_sharp = np.array(([-2, -2, -2], [-2, 17, -2], [-2, -2, -2]), dtype='int')
    #denoised = cv2.fastNlMeansDenoisingColored(img,None,5,5,2,10)
    self.sharp = cv2.filter2D(input_, -1, kernel_sharp)
    return self.sharp

Here is the model where i want to put this layer:

def mymodel300(input_shape=(300, 300, 3)):
    inputs = Input(shape=input_shape)
    # MY CUSTOM LAYER
    sharp = Sharpen(num_outputs=(300,300,3))(inputs)
    # 300x300

    down0 = Conv2D(32, (3, 3), padding='same')(sharp)
    down0 = BatchNormalization()(down0)
    down0 = Activation('relu')(down0)
    down0 = Conv2D(32, (3, 3), padding='same')(down0)
    down0 = BatchNormalization()(down0)
    down0 = Activation('relu')(down0)
    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)

Answer 1

I found out how to create a custom layer with a custom kernel. Thanks apple apple for the hint of using conv2d.

I used Tensorflow's tf.nn.conv2d, instead of cv2.filter2D, with my custom sharpening filter. Here is that custom layer:

# CUSTOM SHARPEN LAYER
class Sharpen(tf.keras.layers.Layer):
    def __init__(self, num_outputs):
        super(Sharpen, self).__init__()
        self.num_outputs = num_outputs

    def build(self, input_shape):
        self.kernel = np.array([[-2, -2, -2], 
                                [-2, 17, -2], 
                                [-2, -2, -2]])
        self.kernel = tf.expand_dims(self.kernel, 0)
        self.kernel = tf.expand_dims(self.kernel, 0)
        self.kernel = tf.cast(self.kernel, tf.float32)

    def call(self, input_):
        return tf.nn.conv2d(input_, self.kernel, strides=[1, 1, 1, 1], padding='SAME')

Answer 2

I think this should be done as part of the preprocessing step as there are no parameters that you would want to train in the sharpening layer in order to aid feature generation or classification, so it does not need to be part of the network model.

Keras Image Pre-processing documentation is here: https://keras.io/preprocessing/image/