How to fix underfitting using a CNN/ why can't my code identify images correctly?

Question

I'm quite new to neural nets and tried writing my own code to classify images. I've been using the Concrete Crack Images for Classification (https://data.mendeley.com/datasets/5y9wdsg2zt/2) to classify whether an image has a crack or is free of defects. From that dataset I randomly extracted 2.000 images, 1.400 for my training set and 300 each for my validation and test set. Half of the images are positive/ show a crack and the other half are negative/ free of defects.

For classification I'm using VGG16 pre-trained on ImageNet. Down below you can see my full code, which I put together using different tutorials that tried to solve a similar task.

Unfortunately it can't identify one single crack image and is classifying everything as negative/ free of defects. I've tried different batch sizes, amounts of epochs, amounts of images, tried it without being pre-trained, but nothing seems to work and I have absolutely no idea why. I'd really appreciate some help so thank you in advance!

If there are any questions left feel free to ask.

import tensorflow as tf
import os
import numpy as np
import keras
from keras.preprocessing import image
from PIL import Image
import os.path, sys
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt

import scipy as sp

from tensorflow.keras.applications import vgg16
from tensorflow.keras.preprocessing.image import load_img, img_to_array, array_to_img, ImageDataGenerator
from tensorflow.keras.models import *
from tensorflow.keras.layers import *
from tensorflow.keras import optimizers
from tensorflow.keras.utils import *

import requests
from io import BytesIO
import random
import pickle
import itertools

from sklearn.metrics import classification_report, confusion_matrix

# load data
PATH = '/Volumes/ConcreteCrackImages'

train_dir = os.path.join(PATH, 'train')
validation_dir = os.path.join(PATH, 'validation')
test_dir = os.path.join(PATH, 'test')

train_negative_dir = os.path.join(train_dir, 'Negative')  # directory with our training negative pictures
train_crack_dir = os.path.join(train_dir, 'Positive')  # directory with our training crack pictures
validation_negative_dir = os.path.join(validation_dir, 'Negative')  # directory with our validation negative pictures
validation_crack_dir = os.path.join(validation_dir, 'Positive')  # directory with our validation crack pictures
test_negative_dir = os.path.join(test_dir, 'Negative')  # directory with our test negative pictures
test_crack_dir = os.path.join(test_dir, 'Positive')  # directory with our test crack pictures

# understand the data

num_negative_tr = len(os.listdir(train_negative_dir))
num_crack_tr = len(os.listdir(train_crack_dir))

num_negative_val = len(os.listdir(validation_negative_dir))
num_crack_val = len(os.listdir(validation_crack_dir))

num_negative_test = len(os.listdir(test_negative_dir))
num_crack_test = len(os.listdir(test_crack_dir))

total_train = num_negative_tr + num_crack_tr
total_val = num_negative_val + num_crack_val
total_test = num_negative_test + num_crack_test

print('total training negative images:', num_negative_tr)
print('total training crack images:', num_crack_tr)

print('total validation negative images:', num_negative_val)
print('total validation crack images:', num_crack_val)

print('total test negative images:', num_negative_test)
print('total test crack images:', num_crack_test)
print("--")
print("Total training images:", total_train)
print("Total validation images:", total_val)
print("Total test images:", total_test)

# variables for pre-processing
batch_size = 32
epochs = 40
IMG_HEIGHT = 224
IMG_WIDTH = 224

# data preparation

train_image_generator = ImageDataGenerator(rescale=1./255) # Generator for our training data
validation_image_generator = ImageDataGenerator(rescale=1./255) # Generator for our validation data
test_image_generator = ImageDataGenerator(rescale=1./255) # Generator for our test data

train_data_gen = train_image_generator.flow_from_directory(batch_size=batch_size,
                                                           directory=train_dir,
                                                           shuffle=True,
                                                           target_size=(IMG_HEIGHT, IMG_WIDTH),
                                                           class_mode='binary')

val_data_gen = validation_image_generator.flow_from_directory(batch_size=batch_size,
                                                              directory=validation_dir,
                                                              target_size=(IMG_HEIGHT, IMG_WIDTH),
                                                              class_mode='binary')

test_data_gen = test_image_generator.flow_from_directory(batch_size=batch_size,
                                                              directory=test_dir,
                                                              target_size=(IMG_HEIGHT, IMG_WIDTH),
                                                              class_mode='binary')

# visualize training images

sample_training_images, _ = next(train_data_gen)

# This function will plot images in the form of a grid with 1 row and 5 columns where images are placed in each column.
# =============================================================================

def plotImages(images_arr):
    fig, axes = plt.subplots(1, 5, figsize=(20,20))
    axes = axes.flatten()
    for img, ax in zip( images_arr, axes):
        ax.imshow(img)
        ax.axis('off')
    plt.tight_layout()
    plt.show()

plotImages(sample_training_images[:5])

# =============================================================================
# create the model/ import vgg16

vgg_conv = vgg16.VGG16(weights='imagenet', include_top=False, input_shape = (224, 224, 3))

# Freeze the layers except the last 4 layers
for layer in vgg_conv.layers[:-8]:
    layer.trainable = False

# Check the trainable status of the individual layers
for layer in vgg_conv.layers:
    print(layer, layer.trainable)

### MODIFY VGG STRUCTURE ###

x = vgg_conv.output
x = GlobalAveragePooling2D()(x)
x = Dense(1, activation="sigmoid")(x)

model = Model(vgg_conv.input, x)
model.compile(loss = "binary_crossentropy", optimizer = optimizers.SGD(lr=0.00001, momentum=0.9), metrics=["accuracy"])

model.summary()

# train the model

history = model.fit(
     train_data_gen,
     steps_per_epoch=total_train // batch_size,
     epochs=epochs,
     validation_data=val_data_gen,
     validation_steps=total_val // batch_size
)


# visualize training results

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']

loss=history.history['loss']
val_loss=history.history['val_loss']

epochs_range = range(epochs)

plt.figure(figsize=(8, 8))
plt.subplot(1, 2, 1)
plt.plot(epochs_range, acc, label='Training Accuracy')
plt.plot(epochs_range, val_acc, label='Validation Accuracy')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')

plt.subplot(1, 2, 2)
plt.plot(epochs_range, loss, label='Training Loss')
plt.plot(epochs_range, val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.show()

# Evaluate the model on the test data using `evaluate`
print('\nEvaluate on test data')
results = model.evaluate(test_data_gen,
                         verbose = 1)
print('test loss, test acc:', results)

#Confusion Matrix and Classification Report
Y_pred = model.predict(test_data_gen)
y_pred = np.argmax(Y_pred, axis=1)
print('Confusion Matrix')
print(confusion_matrix(test_data_gen.classes, y_pred))
print('Classification Report')
target_names = ['Negative', 'Crack']
print(classification_report(test_data_gen.classes, y_pred, target_names=target_names))

Answer 1

There are a couple of issues you can check.

1. since you are using VGG and ImageDataGenerator, you gotta make sure the image data generator do the same preprocessing as the VGG pretrained model required. VGG is trained using the imagenet_utils.preprocessing_input with mode set to "caffe". There are in total three modes, caffe, tf, and torch. Different models are retained with different preprocessing.

2. When you initiate the VGG model, you set the including top to false. And then you get the output of VGG, did a global pool and then added a dense for output. If you dig into the source code of the VGG implementation, the top is just the softmax layer but also FC layers. The FC layers is the way to generate abstraction over the extracted VGG feature. If you don't have enough FC layer, your model is not complicated enough to learn the feature space well.

You can at least try these two to see if they help