CNN return the same result for classification (keras)

Question

I am using CNN to classify two types of pollen: sugi and hinoki. When I used the images taken in visible light as data, it predicted "sugi" for all the test images. In the other hand, when I used images taken in ultraviolet as data, it predicted "hinoki" for all the pics in test set. I have change number of epochs, filter size, batch size, number of channels for several times but the result was the same. What should I do?

Here is my code:

Train program:

import os
from keras.applications.vgg16 import VGG16
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential, Model
from keras.layers import Input, Activation, Dropout, Flatten, Dense, Conv2D, MaxPool2D
#from keras.callbacks import EarlyStoppingByLossVal
from keras.preprocessing.image import ImageDataGenerator
from keras import optimizers
import numpy as np
import time
from PIL import Image 
import csv
import shutil
#import numpy.distutils.system_info as sysinfo
import scipy
import scipy.misc
import matplotlib.pyplot as plt 
import pandas as pd 
# kaneko
from keras.callbacks import TensorBoard


#sysinfo.get_info('lapack')
# 分類するクラス
classes = ['sugi', 'hinoki']
nb_classes = len(classes)


img_width, img_height = 100, 100

# トレーニング用とバリデーション用の画像格納先
train_data_dir = 'cut.kashi/train'
validation_data_dir = 'cut.kashi/validation'

# 今回はトレーニング用に200枚、バリデーション用に50枚の画像を用意した。
nb_train_samples = 1362
nb_validation_samples = 337
#nb_train_samples = 2171
#nb_validation_samples = 528

#batch_size = 64
nb_epoch = 50
gen_tr_batches = 4
folder = './output'
result_dir = 'results'
if not os.path.exists(result_dir):
    os.mkdir(result_dir)
train_imagelist = os.listdir(train_data_dir)

test_list = "./test.train"
font = cv2.FONT_HERSHEY_COMPLEX

def vgg_model_maker():

    model = Sequential()

    model.add(Conv2D(32,5,input_shape=(img_width, img_height,3)))
    model.add(Activation('relu'))
    #model.add(Conv2D(32,5))
    #model.add(Activation('relu'))
    model.add(MaxPool2D(pool_size=(2,2)))

    model.add(Conv2D(64,5))
    model.add(Activation('relu'))
    model.add(MaxPool2D(pool_size=(2,2)))

    model.add(Flatten())
    model.add(Dense(200))
    model.add(Activation('relu'))
    #model.add(Dropout(1.0))

    model.add(Dense(nb_classes, activation='softmax'))


    return model



def image_generator():
    """ ディレクトリ内の画像を読み込んでトレーニングデータとバリデーションデータの作成 """
    train_datagen = ImageDataGenerator(
        rescale=1.0 / 255,
        zoom_range=0.2,
        horizontal_flip=True,
        rotation_range = 180)


    validation_datagen = ImageDataGenerator(rescale=1.0 / 255)

    train_generator = train_datagen.flow_from_directory(
        train_data_dir,
        target_size=(img_width, img_height),
        color_mode='rgb',
        classes=classes,
        class_mode='categorical',
        batch_size=batch_size,
        shuffle=True)

    validation_generator = validation_datagen.flow_from_directory(
        validation_data_dir,
        target_size=(img_width, img_height),
        color_mode='rgb',
        classes=classes,
        class_mode='categorical',
        batch_size=batch_size,
        shuffle=True)

    return (train_generator,validation_generator)


def global_contrast_normalization(filename, s, lmda, epsilon):
    X = numpy.array(Image.open(filename))

    # replacement for the loop
    X_average = numpy.mean(X)
    print('Mean: ', X_average)
    X = X - X_average

    # `su` is here the mean, instead of the sum
    contrast = numpy.sqrt(lmda + numpy.mean(X**2))

    X = s * X / max(contrast, epsilon)

    # scipy can handle it
    scipy.misc.imsave('result.jpg', X)



# Generator for the network's training generator.



# Actual generator for the network's training.


if __name__ == '__main__':
    start = time.time()

    for the_file in os.listdir(folder):
        file_path = os.path.join(folder, the_file)
        try:
            if os.path.isfile(file_path):
                os.unlink(file_path)
        #elif os.path.isdir(file_path): shutil.rmtree(file_path)
        except Exception as e:
            print(e)
     # kaneko
    tensorboard = TensorBoard(log_dir="./kaneko", histogram_freq=0, batch_size= batch_size,write_graph=True)
    # モデル作成
    vgg_model = vgg_model_maker()

    # 最後のconv層の直前までの層をfreeze
    #for layer in vgg_model.layers[:15]:
        #layer.trainable = False

    # 多クラス分類を指定
    vgg_model.compile(loss='categorical_crossentropy',
              optimizer=optimizers.SGD(lr=1e-3, momentum=0.9),
              metrics=['accuracy'])

    # 画像のジェネレータ生成
    train_generator,validation_generator =  image_generator()



    # Fine-tuning
    history_callback = vgg_model.fit_generator(
        train_generator,
        samples_per_epoch=nb_train_samples,
        nb_epoch=nb_epoch,
        validation_data = validation_generator,
        nb_val_samples=nb_validation_samples,
        callbacks=[tensorboard])

    loss_history = history_callback.history["loss"]
    accuracy_history = history_callback.history["acc"]
    val_loss_history = history_callback.history["val_loss"]
    val_accuracy_history = history_callback.history["val_acc"]
    numpy_loss_history = np.array(loss_history)
    numpy_accuracy_history = np.array(accuracy_history)
    numpy_val_loss_history = np.array(val_loss_history)
    numpy_val_accuracy_history = np.array(val_accuracy_history)

    f = open("results/result.csv","w")
    writer = csv.writer(f)
    writer.writerow(["loss","accuracy","validation loss","validation accuracy"])
    for j in range(len(numpy_loss_history)):
        writer.writerow([numpy_loss_history[j],numpy_accuracy_history[j],numpy_val_loss_history[j],numpy_val_accuracy_history[j]])
    epochnum = range(len(numpy_loss_history))
    print(len(epochnum))
    #plt.plot(epochnum,numpy_loss_history, label = "loss")
    #plt.legend()
    plt.plot(loss_history)
    plt.plot(val_loss_history)
    plt.legend(['loss', 'val_loss'])
    plt.show()
    #plt.savefig("./Documents/Ghi1/shigaisen_loss.png")
    plt.clf()
    plt.plot(epochnum,numpy_accuracy_history, label = "accuracy")
    plt.show()
    #plt.savefig(".../Documents/Ghi1/shigaisen_accuracy.png")
    plt.clf()
    vgg_model.save_weights(os.path.join(result_dir, 'finetuning.h5'))

    process_time = (time.time() - start) / 60
    print(u'学習終了。かかった時間は', process_time, u'分です。')

Test program:

import os, sys
import numpy as np
import cv2

from keras.applications.vgg16 import VGG16
from keras.models import Sequential, Model
from keras.layers import Input, Activation, Dropout, Flatten, Dense, Conv2D,MaxPool2D
from keras.preprocessing import image
from keras.preprocessing.image import ImageDataGenerator
from keras import optimizers
from datetime import datetime

classes = ['sugi', 'hinoki']
nb_classes = len(classes)
img_width, img_height = 100, 100
DataShape = (100,100,3)
result_dir = 'results'
#test_list = "./testfile"
test_list = "./test.train"
font = cv2.FONT_HERSHEY_COMPLEX
# このディレクトリにテストしたい画像を格納しておく
test_data_dir = 'cut/test'
folder = './output'
def model_load():
    # VGG16, FC層は不要なので include_top=False
    model = Sequential()

    model.add(Conv2D(32,5,input_shape=(img_width, img_height,3)))
    model.add(Activation('relu'))
    #model.add(Conv2D(32,5))
    #model.add(Activation('relu'))
    model.add(MaxPool2D(pool_size=(2,2)))

    model.add(Conv2D(64,5))
    model.add(Activation('relu'))
    model.add(MaxPool2D(pool_size=(2,2)))

    model.add(Flatten())
    model.add(Dense(200))
    model.add(Activation('relu'))
    #model.add(Dropout(1.0))

    model.add(Dense(nb_classes, activation='softmax'))

    #adam = Adam(lr=1e-4)

    # 学習済みの重みをロード
    model.load_weights(os.path.join(result_dir, 'finetuning.h5'))

    # 多クラス分類を指定
    model.compile(loss='categorical_crossentropy',
              optimizer=optimizers.SGD(lr=1e-3, momentum=0.9),
              metrics=['accuracy'])

    return model

def image_generator():
    """ ディレクトリ内の画像を読み込んでトレーニングデータとバリデーションデータの作成 """
    test_datagen = ImageDataGenerator(
        rescale=1.0 / 255,
        zoom_range=0.2,
        horizontal_flip=True,
        rotation_range = 180)

    #validation_datagen = ImageDataGenerator(rescale=1.0 / 255)

    test_generator = test_datagen.flow_from_directory(
        test_data_dir,
        target_size=(img_width, img_height),
        color_mode='rgb',
        classes=classes,
        class_mode='categorical',
        batch_size=batch_size,
        shuffle=True)

def image_resize(image, width = None, height = None, inter = cv2.INTER_AREA):
    # initialize the dimensions of the image to be resized and
    # grab the image size
    dim = None
    (h, w) = image.shape[:2]

    # if both the width and height are None, then return the
    # original image
    if width is None and height is None:
        return image

    # check to see if the width is None
    if width is None:
        # calculate the ratio of the height and construct the
        # dimensions
        r = height / float(h)
        dim = (int(w * r), height)

    # otherwise, the height is None
    else:
        # calculate the ratio of the width and construct the
        # dimensions
        r = width / float(w)
        dim = (width, int(h * r))

    # resize the image
    resized = cv2.resize(image, dim, interpolation = inter)

    # return the resized image
    return resized

def test(model,path,filename,sugi):
    test_imagelist = []

    # テスト用画像取得
    #test_imagelist = os.listdir(test_data_dir)

    #test_imagelist = os.listdir(test_data_dir)

    iml = cv2.imread(path,cv2.IMREAD_COLOR)


    img = image_resize(iml,height=960)
    img_array = np.array(img)

    cimg = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    cimg = cv2.medianBlur(cimg,5)

    #_,cimg = cv2.threshold(cimg,0,255,cv2.THRESH_BINARY| cv2.THRESH_OTSU)
    #cv2.imwrite(datetime.now().strftime('%s')+"binary.jpg",cimg)
    #sys.exit()

    circles = cv2.HoughCircles(cimg,cv2.HOUGH_GRADIENT,1,10,param1=15,param2=20,minRadius=10,maxRadius=25)
    circles = np.uint16(np.around(circles))[0,:]
    print (len(circles))
    center = []
    predict = []
    for i in circles:
        half = DataShape[0]//2
        zoom_data = img_array[i[1]-half:i[1]+half,i[0]-half:i[0]+half,:]
        if zoom_data.shape!=DataShape : continue
        czoom = cv2.cvtColor(zoom_data, cv2.COLOR_BGR2GRAY)
        czoomarr = np.array(zoom_data)
        cen = czoom[half,half]
        #edge = czoom[0,0]
        if cen != 0:
        #if cen < 255:
        #if czoom[30,30] < 80:
            test_imagelist.append(zoom_data)
            center.append(i)
        label_num = len(test_imagelist)

    print(len(center))
    print(label_num)

    for im in test_imagelist:
        x = image.img_to_array(im)
        x = np.expand_dims(x, axis=0)
        # 学習時に正規化してるので、ここでも正規化
        x = x / 255
        pred = model.predict(x)[0]
        print(pred)
        predict.append(pred)
    TP = 0
    TN = 0
    FN = 0
    FP = 0
    for j in range(label_num):
        if predict[j][0] > predict[j][1]:
            if sugi == 1:
                #TP+=1
                TN+=1
            else:
                #FP+=1
                FN+=1
            #cv2.circle(img,(center[j][0],center[j][1]),center[j][2],(0,255,0),2)
            cv2.putText(img,'S',(center[j][0],center[j][1]), font, 0.5,(0,255,0),1,cv2.LINE_AA)
        if predict[j][0] < predict[j][1]:
            #cv2.circle(img,(center[j][0],center[j][1]),center[j][2],(0,0,255),2)
            if sugi == 1:
                #FN+=1
                FP+=1
            else:
                #TN+=1
                TP+=1
            cv2.putText(img,'H',(center[j][0],center[j][1]), font,0.5,(0,0,255),1,cv2.LINE_AA)

    cv2.imwrite("output/"+"output"+filename,img)

    return TP, FP, FN, TN


if __name__ == '__main__':

    # モデルのロード
    TP,FP,FN,TN = 0,0,0,0
    print(TP,FP,FN,TN) 
    sugi = 0
    c = "ス"
    model = model_load()
    for the_file in os.listdir(folder):
        file_path = os.path.join(folder, the_file)
        try:
            if os.path.isfile(file_path):
                os.unlink(file_path)
        #elif os.path.isdir(file_path): shutil.rmtree(file_path)
        except Exception as e:
            print(e)

    for the_file in os.listdir(test_list):
        #print(the_file)
        if c in the_file:
            sugi = 1
        else:
            sugi = 0
        file_path = os.path.join(test_list, the_file)
        tp1,fp1,fn1,tn1 = test(model,file_path,the_file,sugi)
        TP += tp1
        FP += fp1
        FN += fn1
        TN += tn1


    precision = TP/(TP + FP)
    recall = TP/(TP + FN)
    F = (2*recall*precision)/(recall + precision)
    #cv2.imwrite("output/" + "result.jpg",img)

    print("TP = %lf, TN = %lf, FN = %lf, FP = %lf" %(TP,TN,FN,FP))
    print("precision = %lf, recall = %lf" %(precision,recall))
    print("F measure = %lf" %(F))

Answer 1

One problem I can see is here x = x / 255 in test method. You need to get float values for proper normalisation. I faced the same issue and proper scaling got it working. Here's the link

I hope this helps.

EDIT: My answer is considering for python 2.

Answer 2

I suspect you got a wrong folder structure.

The ImageDataGenerator will create classes based on the folder structure you use.

You should have inside your "datadir":

• One "sugi" folder with all sugi images
• One "hinoki" folder with all hinoki images

But it seems you have instead:

• One "visible" folder
• One "ultraviolet" folder

This will certainly make the generator think "visible=sugi" and "ultraviolet=hinoki".