Generator Stalls Keras fit_generator

Question

I have a data set with 9 columns, the last is the target variable as a csv with header. I am trying to write a generator to train a model in keras. The code is below. The training run during the first epoch but stalls / hangs for ever before finishing.

from sklearn.datasets import california_housing
import pandas as pd
import numpy as np

data=california_housing.fetch_california_housing()
cols=data.feature_names
cols.append('y')
data=pd.DataFrame(np.column_stack([data.data,data.target.reshape((data.target.shape[0],1))]),columns=cols)
data.to_csv('/media/jma/DATA/calhousing.csv',index=False)

Code for the generator:

import csv 
import numpy as np 

def generate_arrays_from_file(file_name,batchsz):
    csvfile = open(file_name)
    reader = csv.reader(csvfile)

    batchCount = 0

    inputs = []
    targets = []


    while True: #infinite loop

        linecounter=0 #which line the reader is reading

        for line in reader:

            if linecounter >0: #is not the header

                inputs.append(line[0:8])
                targets.append(line[8])

                batchCount += 1 # we added 

                if batchCount >= batchsz: # we have our mini batch

                    batchCount = 0 #reset batch counter

                    X = np.array(inputs,dtype="float32")
                    y = np.array(targets,dtype="float32")

                    yield (X, y)

                    #reset the lists to hold the batches
                    inputs = [] 
                    targets = []

            linecounter += 1 #increment the line read

      linecounter = 0 #reset      

Running like this:

from keras.models import Sequential
from keras.layers import Dense


batch_size =100

train_gen=generate_arrays_from_file('/media/jma/DATA/calhousing.csv',batchsz=batch_size)


model = Sequential()
model.add(Dense(32, input_shape=(8,)))
model.add(Dense(1, activation='linear'))
model.compile(optimizer='rmsprop',
              loss='mse', metrics=['mse'])


model.fit_generator(train_gen,steps_per_epoch=data.shape[0] / batch_size, epochs=5, verbose=1)

Epoch 1/5 194/206 [===========================>..] - ETA: 0s - loss: 67100.1775 - mean_squared_error: 67100.1775

Answer 1

What OP changed:

import csv 
import numpy as np 

def generate_arrays_from_file(file_name,batchsz):

    ###################
    ### Moved this: ###
    ###################
    # csvfile = open(file_name)
    # reader = csv.reader(csvfile)
    ### End ###########

    batchCount = 0

    inputs = []
    targets = []

    linecounter=0 #which line the reader is reading

    while True: #infinite loop
        ################
        ### to here: ###
        ################
        with open(file_name, "r") as csvfile: 
            for line in csv.reader(csvfile):  
                ### End ###########

                if linecounter >0: #is not the header

                    #could procress data as well
                    inputs.append(line[0:8])
                    targets.append(line[8])

                    batchCount += 1 # we added 

                    if batchCount >= batchsz: # we have our mini batch
                        batchCount = 0 #reset batch counter
                        X = np.array(inputs,dtype="float32")
                        y = np.array(targets,dtype="float32")
                        yield (X, y)

                        #reset the lists to hold the batches
                        inputs = [] 
                        targets = []

            linecounter += 1 #increment the line read
        linecounter = 0

Answer 2

I found the issue and am posting this for anyone else that is looking for an example:

import csv 
import numpy as np 

def generate_arrays_from_file(file_name,batchsz):

    batchCount = 0

    inputs = []
    targets = []

    linecounter=0 #which line the reader is reading

    while True: #infinite loop
        with open(file_name, "r") as csvfile: 
            for line in csv.reader(csvfile):  

                if linecounter >0: #is not the header

                    #could procress data as well
                    inputs.append(line[0:8])
                    targets.append(line[8])

                    batchCount += 1 # we added 

                    if batchCount >= batchsz: # we have our mini batch
                        batchCount = 0 #reset batch counter
                        X = np.array(inputs,dtype="float32")
                        y = np.array(targets,dtype="float32")
                        yield (X, y)

                        #reset the lists to hold the batches
                        inputs = [] 
                        targets = []

                linecounter += 1 #increment the line read
            linecounter = 0