datagenerator made on python whit keras is too slow, how to improve it speed?

Question

i made a datagenerator for a multi input model using this two references

https://datascience.stackexchange.com/questions/47623/how-feed-a-numpy-array-in-batches-in-keras

Keras open images and create a batch with them

so my generator looks like this

def load_data(df,path,position):
    X_numerical = []
    X_img = []
    Y = []

    for i in df.index.values:
        # read one or more samples from your storage, do pre-processing, etc.
        # for example:
        basePath = os.path.sep.join([path, "{}-*".format(i)])
        LiqPaths = sorted(list(glob.glob(basePath)))
        img = tf.keras.preprocessing.image.load_img(LiqPaths[0], target_size=(100, 100))
        img = tf.keras.preprocessing.image.img_to_array(img)
        img = np.expand_dims(img, axis=0)
        
        x=np.asarray(df.iloc[i]).astype('float32')
        x=tf.expand_dims(x, axis=-1)
        
        y = position[i]

        
        X_numerical.append(x)
        X_img.append(img)
        Y.append(y)
    return [np.array(X_numerical), np.array(X_img)], np.array(Y)

this is the generator

def batch_generator(df, path, position, batch_size):
    batch=[]
  #  print(len(df.index.values))
    while True:
            for i in df.index.values:
                batch.append(i)
                if len(batch)==batch_size:
                    yield load_data(df,path,position)
           batch=[]

and my model is this one

input_data = Input(name="input_numerical",shape=(2,1,))

x1  = LSTM(
        units = 1024, 
        return_sequences=True
        )(input_data)
x1  = LSTM(
        units = 1024, 
        return_sequences=False
        )(x1)


input_img = Input(name="input_images",shape=(60,30,3,))

x2 = Conv2D(128, (3, 3), name="first_conv", activation='relu', input_shape=(60, 30, 3))(input_img)
x2 =MaxPooling2D((2, 2),name="first_pooling")(x2)
x2 =Conv2D(256,  (3, 3),name="second_conv", activation='relu')(x2)
x2 =MaxPooling2D((2, 2),name="seccond_pooling")(x2)
x2 = Flatten(name="flatten",)(x2)
x2 =Dense(1024, name="first_dense", activation='relu')(x2)

merge1  =concatenate([x1,x2], axis = 1)
x3 =Dense(1024,name="seccond_dense", activation='relu')(merge1)
output_liq =Dense(1,name="output", activation='sigmoid')(x3)

model = Model([input_img,input_data], output_liq)
model.summary()

tf.keras.utils.plot_model(
    model, show_shapes=True
)

model.compile(
              loss="BinaryCrossentropy", 
              optimizer=tf.keras.optimizers.Nadam(learning_rate=1e-6),
              metrics=["BinaryAccuracy"])

history = model.fit(gen,
                         epochs= 30, 
                         shuffle=False,
                         batch_size=256
                         #validation_data=([val_img, test_numerical1], val_y)
                         )

but it is to slow, to much i think, so i would like to know if you know how to make it perform faster, my dataset is about 44k pictures and a pandas whit 44k rows whit 2 features, i hope you can helpme

Answer 1

I tried creating some dummy images and a much smaller model (shown below) to try and compare performances. I took only 10 images to send through the gen function you built, and it took about 8 minutes to run. I'm not 100% sure why the gen function is this slow. It probably has to do with the for loops.

img_paths = np.arange(10)
x_nums = np.random.randn(10,2).tolist()
positions = np.arange(10)

df = pd.DataFrame({'img_paths':img_paths,
                   'nums':x_nums,
                   })

path = '/content/drive/MyDrive/Stack Overflow/Images - 60,30,3/'
gen = batch_generator(df,path,positions,5)

---

I tried to recreate your workflow to an extent to use tensorflow-datasets. I used a smaller model with 1.8 million parameters instead of 35 million as defined by you

input_data = tf.keras.layers.Input(name="input_numerical",shape=(2,1,))

x1  = tf.keras.layers.LSTM(
        units = 128, 
        return_sequences=True
        )(input_data)
x1  = tf.keras.layers.LSTM(
        units = 128, 
        return_sequences=False
        )(x1)


input_img = tf.keras.layers.Input(name="input_images",shape=(60,30,3,))

x2 = tf.keras.layers.Conv2D(128, (3, 3), name="first_conv", activation='relu', input_shape=(60, 30, 3))(input_img)
x2 = tf.keras.layers.MaxPooling2D((2, 2),name="first_pooling")(x2)
x2 = tf.keras.layers.Conv2D(256,  (3, 3),name="second_conv", activation='relu')(x2)
x2 = tf.keras.layers.MaxPooling2D((2, 2),name="seccond_pooling")(x2)
x2 = tf.keras.layers.Flatten(name="flatten",)(x2)
x2 = tf.keras.layers.Dense(64, name="first_dense", activation='relu')(x2)

merge1  = tf.keras.layers.Concatenate(axis = 1)([x1,x2])
x3 = tf.keras.layers.Dense(64,name="seccond_dense", activation='relu')(merge1)
output_liq = tf.keras.layers.Dense(1,name="output", activation='sigmoid')(x3)

model = tf.keras.models.Model([input_img,input_data], output_liq)
model.summary()

The total run time of tf.data.dataset was 35 seconds for 1 epoch having 10,000 images compared to 8 minutes for 10 images using the gen function

---

Here I have created the necessary inputs as best as I could. I do not understand how exactly you are getting the images, but I have just put the image paths inside the dataframe

path = '/content/drive/MyDrive/Stack Overflow/Images - 60,30,3/'

img_paths = os.listdir(path)
x_nums = np.random.randn(len(img_paths),2).tolist()
positions = np.random.randint(0,2,len(img_paths))


df = pd.DataFrame({'img_paths':img_paths,
                   'nums':x_nums,
                   })

---

Next we create the tensorflow dataset

def preprocess(x,y):
  img = tf.io.read_file(x[0])
  img = tf.io.decode_png(img, channels=3)
  img = tf.cast(img,dtype=tf.float32)
  img = img / 255.

  x_num = tf.expand_dims(x[1],1) #You might have to change this line depending on how your data looks
  return (img,x_num),y

def dataset_gen(df,basepath,positions,batch_size):
  img_paths = tf.constant([basepath+i for i in df.img_paths])
  x_numerical = tf.constant([np.array(x) for x in df.nums.values])
  
  train_dataset = tf.data.Dataset.from_tensor_slices(((img_paths,x_numerical),positions))
  train_dataset = train_dataset.map(preprocess)
  train_dataset = train_dataset.batch(batch_size)

  return train_dataset

train_data = dataset_gen(df,path,positions,128)

You can then just plug this data into the model

model.compile(
              loss="BinaryCrossentropy", 
              optimizer=tf.keras.optimizers.Nadam(learning_rate=1e-6),
              metrics=["BinaryAccuracy"])

history = model.fit(train_data,
                         epochs= 1
                         #validation_data=([val_img, test_numerical1], val_y)
                         )