Why do I keep getting an error saying "maximum recursion depth exceeded while calling a Python object" in Keras from Tensorflow 2.0?

Question

I am trying to train a stacked neural network architecture with CNNs, GRUs and a CTC in tensorflow 2.0's edition of Keras. I keep getting an error saying "RecursionError: maximum recursion depth exceeded while calling a Python object".

I have tried importing sys and setting the Recursion Limit to be very high using sys.setrecursionlimit() but the program just stops running.

import sys
import tensorflow as tf
from generator_tf2 import VideoGenerator
from network_model_GRU_tf2 import Decoder
from helpers import labels_to_text
from spell import Spell
from network_model_GRU_tf2 import Network_Model
from keras.callbacks import EarlyStopping, TensorBoard, CSVLogger, ModelCheckpoint
import numpy as np
import datetime
import os
import matplotlib.pyplot as plt
from tensorflow.compat.v1 import ConfigProto
from tensorflow.compat.v1 import InteractiveSession

config = ConfigProto()
config.gpu_options.allow_growth = True
session = InteractiveSession(config=config)

# strategy = tf.distribute.MirroredStrategy()
strategy = tf.distribute.MirroredStrategy(cross_device_ops=tf.distribute.HierarchicalCopyAllReduce())

print('Number of devices: {}'.format(strategy.num_replicas_in_sync))

PREDICT_GREEDY      = False #Use of Greedy Search
PREDICT_BEAM_WIDTH  = 200 #Set Beam search width
MAX_STRING_LENGTH   = 114 #Maximum sentence length
MAX_VIDEO_LENGTH    = 114 #Maximum number of video frames
start_epoch         = 0

#Directories
PREDICT_DICTIONARY = os.path.join(r"F:\Lip Reading System\vsnet","LessFourSecondsSentences.txt") #Needed for Curriculum learning
INPUT_DIR = os.path.join(r"F:\Lip Reading System\vsnet","models_06082019") #Keras model directory
OUTPUT_DIR = os.path.join(r"F:\Lip Reading System\vsnet","models_08082019") #Keras model directory

#Generator for training
lip_gen_train = VideoGenerator("LessFourSeconds.txt","LessFourSeconds_videoframes_training","LessFourSeconds_subtitles_training", 30, absolute_max_video_len=MAX_VIDEO_LENGTH, absolute_max_string_len=MAX_STRING_LENGTH)
lip_gen_train.build_data_from_frames()

#Generator for testing
lip_gen_test = VideoGenerator("testing_videos.txt","testing_videoframes","testing_subtitles", 10, absolute_max_video_len=MAX_VIDEO_LENGTH, absolute_max_string_len=MAX_STRING_LENGTH)
lip_gen_test.build_data_from_frames()

#Set neural network conditions network
with strategy.scope():
    network_model = Network_Model(img_c=1, img_w=100, img_h=50, frames_n=MAX_VIDEO_LENGTH, absolute_max_string_len=MAX_STRING_LENGTH, output_size=38)
    network_model.summary()
    adam = tf.keras.optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
    # load weight if necessary
    if start_epoch > 0:
        weight_file = os.path.join(INPUT_DIR, 'weights%02d.h5' % (start_epoch))
        network_model.model.load_weights(weight_file)

# the loss calc occurs elsewhere, so use a dummy lambda func for the loss
network_model.model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=adam, metrics=['accuracy'])

#Spelling and Decoder
spell = Spell(path=PREDICT_DICTIONARY)
decoder = Decoder(greedy=PREDICT_GREEDY, beam_width=PREDICT_BEAM_WIDTH,postprocessors=[labels_to_text, spell.sentence])

#Early stop and function to save weights
early_stop = tf.keras.callbacks.EarlyStopping(monitor='loss', min_delta=0.001, patience=4, mode='min', verbose=1)
checkpoint = tf.keras.callbacks.ModelCheckpoint(os.path.join(OUTPUT_DIR, "weights{epoch:02d}.h5"), monitor='val_loss', save_weights_only=True, mode='min', period=10)

#Generator
train_history = network_model.model.fit_generator(generator=lip_gen_train.get_batch(),
                        steps_per_epoch=lip_gen_train.video_dataset_steps,
                        epochs=1000,
                        validation_data=lip_gen_test.get_batch(),
                        validation_steps=lip_gen_test.video_dataset_steps)

The script works fine when executed in tensorflow 1.10.0 with keras 2.2.4 and does not produce the error that I keep getting below:

Traceback (most recent call last): File "F:\Lip Reading System\vsnet\training_tf2.py", line 71, in validation_steps=lip_gen_test.video_dataset_steps) ...... RecursionError: maximum recursion depth exceeded while calling a Python object

Answer 1

include model.compile() inside the strategy.scope()

#Set neural network conditions network
with strategy.scope():
    network_model = Network_Model(img_c=1, img_w=100, img_h=50, frames_n=MAX_VIDEO_LENGTH, absolute_max_string_len=MAX_STRING_LENGTH, output_size=38)
    network_model.summary()
    adam = tf.keras.optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
    # load weight if necessary
    if start_epoch > 0:
        weight_file = os.path.join(INPUT_DIR, 'weights%02d.h5' % (start_epoch))
        network_model.model.load_weights(weight_file)

     # the loss calc occurs elsewhere, so use a dummy lambda func for the loss
     network_model.model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=adam, metrics=['accuracy'])