How can I reduce the loss rate of G and D in GAN?

Question

I'm building a GAN using Tensorflow.

At first, I created a GAN that produces a 32x32 image.

Modified the model to add a layer to create a 128x128 image.

By the way, 32x32 GAN G, D loss value was ok, but the loss value is very high as the layer size and image size are increased.

I modified the layer and modified other hyper parameters to lower the loss, but it's still high.

I wonder how to reduce the loss of G and D.

import os.path
import numpy as np
from keras.models import *
from keras.layers import *
from keras.optimizers import *
from keras.layers.convolutional import Conv2DTranspose, MaxPooling2D, UpSampling2D, Conv2D
from keras.layers.core import Reshape, Dense, Dropout, Flatten
from keras.layers.advanced_activations import LeakyReLU, ReLU
from keras.layers.normalization import BatchNormalization
import keras.backend as K
import matplotlib.pyplot as plt
from PIL import Image
from keras.models import load_model
K.set_image_data_format('channels_last')

class Gan:
    def __init__(self,img_data):
        img_size = img_data.shape[1]
        channel = img_data.shape[3] if len(img_data.shape) >= 4 else 1

        self.img_data = img_data
        self.input_shape = (img_size,img_size,channel)

        self.img_rows = img_size
        self.img_cols = img_size
        self.channel = channel
        self.noise_size = 128

        self.create_d()
        self.create_g()

        optimizer = Adam(lr=0.0008)
        self.D.compile(loss='binary_crossentropy', optimizer=optimizer)

        optimizer = Adam(lr=0.0004)
        self.D.trainable = False
        self.AM = Sequential()
        self.AM.add(self.G)
        self.AM.add(self.D)
        self.AM.compile(loss='binary_crossentropy',optimizer=optimizer)

    def create_g(self):
        self.G = Sequential()
        dropout = 0.4

        self.G.add(Dense(8 * 8 * 1024, input_dim=self.noise_size))
        self.D.add(Dropout(dropout))
        self.G.add(Activation('relu'))
        self.G.add(Reshape((8, 8, 1024)))
        self.G.add(Dropout(dropout))
        self.G.add(Conv2DTranspose(512, 5, strides=2, padding ='same'))
        self.D.add(Dropout(dropout))
        self.G.add(Activation('relu'))
        self.G.add(Conv2DTranspose(256, 5, strides=2, padding ='same'))
        self.D.add(Dropout(dropout))
        self.G.add(Activation('relu'))
        self.G.add(Conv2DTranspose(128, 5, strides=2, padding ='same'))
        self.D.add(Dropout(dropout))
        self.G.add(Activation('relu'))
        self.G.add(Conv2DTranspose(64, 5, strides=2, padding='same'))
        self.D.add(Dropout(dropout))
        self.G.add(Activation('relu'))
        self.G.add(Conv2DTranspose(self.channel, 5, strides =1,padding='same'))
        self.G.add(Activation('sigmoid'))
        self.G.summary()
        return self.G

    def create_d(self):
        self.D = Sequential()
        dropout = 0.4
        self.D.add(Conv2D(64, 5, strides=2, input_shape=self.input_shape, padding='same'))
        self.D.add(LeakyReLU(alpha=0.2))
        self.D.add(Dropout(dropout))
        self.D.add(BatchNormalization(momentum=0.9))
        self.D.add(Conv2D(128, 5, strides=2, input_shape=self.input_shape, padding='same'))
        self.D.add(LeakyReLU(alpha=0.2))
        self.D.add(Dropout(dropout))
        self.D.add(Conv2D(256, 5, strides=2, input_shape=self.input_shape, padding='same'))
        self.D.add(LeakyReLU(alpha=0.2))
        self.D.add(Dropout(dropout))
        self.D.add(Conv2D(512, 5, strides=1, input_shape=self.input_shape, padding='same'))
        self.D.add(LeakyReLU(alpha=0.2))
        self.D.add(Dropout(dropout))
        self.D.add(Conv2D(1024, 5, strides=2, input_shape=self.input_shape, padding='same'))
        self.D.add(LeakyReLU(alpha=0.2))
        self.D.add(Dropout(dropout))
        self.D.add(Flatten())
        self.D.add(Dense(1))
        self.D.add(Activation('sigmoid'))
        self.D.summary()
        return self.D


    def train(self, sess, batch_size=100):

        images_train = self.img_data[np.random.randint(0, self.img_data.shape[0], size=batch_size), :, :, :] #shape[0] -> image data의 숫자
        noise = np.random.uniform(-1.0,1.0, size=[batch_size,self.noise_size])
        images_fake = self.G.predict(noise)

        x = np.concatenate((images_train, images_fake))
        y = np.ones([2*batch_size,1])
        y[batch_size:,:] = 0
        self.D.trainable = True
        d_loss = self.D.train_on_batch(x,y)

        y = np.ones([batch_size,1])
        noise = np.random.uniform(-1.0,1.0,size=[batch_size,self.noise_size])
        self.D.trainable = False
        a_loss = self.AM.train_on_batch(noise,y)

        return d_loss, a_loss, images_fake

    def save_weigths(self):

        self.G.save_weights('gan_g_weights')
        self.D.save_weights('gan_d_weights')


    def load(self):
        if os.path.isfile('gan_g_weights'):
            self.G.load_weights('gan_g_weights')
            print("Load G from file")
        if os.path.isfile('gan_d_weights'):
            self.D.load_weights('gan_d_weights')
            print("Load D from file")

class faceData():
    def __init__(self):
        img_data_list = []
        images = os.listdir("data_rgb1")

        for path in images:
            img = Image.open("data_rgb1/" + path)
            img_data_list.append([np.array(img).astype('float32')])

        self.x_train = np.vstack(img_data_list) / 255.0
        print(self.x_train.shape)

dataset = faceData()
x_train =dataset.x_train

gan = Gan(x_train)
gan.load()

sess = tf.Session()
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())

epochs = 1000
sample_size = 10
batch_size = 50
train_per_epoch = x_train.shape[0] // batch_size



for epoch in range(0,epochs):
    total_d_loss = 0.0
    total_a_loss = 0.0
    imgs = None

    for batch in range(0, train_per_epoch):
        d_loss, a_loss, t_imgs = gan.train(batch_size)
        total_d_loss += d_loss
        total_a_loss += a_loss
        if imgs is None:
            imgs = t_imgs


    total_d_loss /= train_per_epoch
    total_a_loss /= train_per_epoch
    print("Epoch: {}, D Loss: {}, AM loss: {} " .format(epoch, total_d_loss, total_a_loss))
    fig, ax = plt.subplots(1, sample_size, figsize = (sample_size, 1))
    if epoch == 999:
        for i in range(0, sample_size):
            ax[i].set_axis_off()
            ax[i].imshow(imgs[i].reshape((gan.img_rows, gan.img_cols, gan.channel)), interpolation='nearest');
            plt.savefig('result%d.png' % epoch)
        saver.save(sess, os.path.join('save', 'model_{}'.format(epoch)))


    plt.close('all')
    gan.save_weigths()

Result:

Epoch: 0, D Loss: 8.065221479096389, AM loss: 14.922738138189171

Epoch: 1, D Loss: 8.052544213793604, AM loss: 14.836829509831928

Epoch: 2, D Loss: 8.02602034776949, AM loss: 14.889192866794954

Epoch: 3, D Loss: 8.05762272074743, AM loss: 14.88101108667209

Epoch: 4, D Loss: 8.045719083795692, AM loss: 14.863829361000642

Epoch: 5, D Loss: 8.052135099614333, AM loss: 14.872829325913173

Epoch: 6, D Loss: 8.026918762226396, AM loss: 14.900647337666623

Epoch: 7, D Loss: 8.091860083759133, AM loss: 14.836829485626994

Epoch: 8, D Loss: 8.05686701130746, AM loss: 14.935828973799188

Epoch: 9, D Loss: 8.038368832641448, AM loss: 14.832738677862332

Epoch: 10, D Loss: 8.06173144016169, AM loss: 14.904738174477204

Epoch: 11, D Loss: 8.032495556749064, AM loss: 14.926010857983893 . . .

Answer 1

Welcome to the beatiful world of training a GAN. It is not an easy task yet not impossible. I cannot tell what the problems are just by looking at the loss from the first 10 epochs.

I highly recommend you to take a look at this article. You are probably having trouble since the equilibrium must have been broken when you added more layers.

Some tips recommended in the article are:

1. Look at the gradients.
2. Perform a pretraining.
3. Do not use hard labels.

Some personal tips are: add batch normalization at the generator. Do not use dense layers in the generator.

Good luck, have fun and please! Post some of the images you are generating I am very curious!