TF: how to solve ValueError: Variable ...weight already exists, disallowed. Did you mean to set reuse=True

Question

I build an Inverse Compositional CNN but it reports errors as bellows:

ValueError: Variable left_src_tgt_warp/ICSTN/icnv1/weight already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:

I find that using tf.reset_default_graph() can solve this problem. But I do not know where I should add it.

for l in range(opt.warpN):
    with tf.variable_scope("ICSTN", reuse=l > 0) as sc:
    end_points_collection = sc.original_name_scope + '_end_points'
    with slim.arg_scope([slim.conv2d, slim.conv2d_transpose],
                       normalizer_fn=slim.batch_norm,
                      weights_regularizer=slim.l2_regularizer(0.05),
                            normalizer_params=batch_norm_params,
                            activation_fn=tf.nn.relu,
                          outputs_collections=end_points_collection):
            imageWarp = inverse_warp(
                inputImage,
                depth,
                pM,
                intrinsics,
                intrinsics_inv)
            imageWarpAll.append(imageWarp)
            feat = tf.reshape(imageWarp, [batch_size, H, W, C])
            print('feat shape:', feat.get_shape())
            print('pM_ini:', pM.get_shape())
            with tf.variable_scope("icnv1"):
                feat = conv2Layer(opt, feat, 4)
                feat = tf.nn.relu(feat)
            with tf.variable_scope("icnv2"):
                feat = conv2Layer(opt, feat, 8)
                feat = tf.nn.relu(feat)
                feat = tf.nn.max_pool(feat, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="VALID")
            feat = tf.reshape(feat, [opt.batch_size, -1])
            with tf.variable_scope("fc3"):
                feat = linearLayer(opt, feat, 48)
                feat = tf.nn.relu(feat)
            with tf.variable_scope("fc4"):
                feat = linearLayer(opt, feat, 6, final=True)
            dp = tf.reshape(feat, [-1, 6])
            print('dp: ', dp.get_shape())
        dpM = pose_vec2mat(dp)
        pM = tf.matmul(dpM, pM)
    imageWarp = inverse_warp(
        inputImage,
        depth,
        pM,
        intrinsics,
        intrinsics_inv)
    imageWarpAll.append(imageWarp)
    return imageWarpAll, pM
def build_train_graph():
    with tf.name_scope("cnn1"):...
    with tf.name_scope("cnn2"):...
    with tf.name_scope("Inverse Compositional CNN"):...
def train(self, opt):
    with tf.variable_scope(tf.get_variable_scope()):
            for i in range(opt.num_gpus):
                print('gpu:', i)
                with tf.device('/gpu:%d' % i):
                    self.build_train_graph(L_img_splits[i], R_img_splits[i], L_cam2pix_splits[i], L_pix2cam_splits[i],
                                       R_cam2pix_splits[i], R_pix2cam_splits[i], L_sca_splits[i], R_sca_splits[i],
                                       reuse_variables)
                    self.collect_summaries(i)
                    tower_losses.append(self.total_loss)
                    reuse_variables = True
                    grads = opt_step.compute_gradients(self.total_loss)
                    tower_grads.append(grads)
        grads = average_gradients(tower_grads)
        apply_gradient_op = opt_step.apply_gr`enter code here`adients(grads, global_step=global_step)
        incr_global_step = tf.assign(global_step, global_step + 1)
        total_loss = tf.reduce_mean(tower_losses)

        tf.summary.scalar('learning_rate', learning_rate, ['model_0'])
        tf.summary.scalar('total_loss', total_loss, ['model_0'])
        summary_op = tf.summary.merge_all('model_0')
        # self.collect_summaries()
        # SESSION
        config = tf.ConfigProto(allow_soft_placement=True)
        config.gpu_options.allow_growth = True
        sess = tf.Session(config=config)

        # SAVER
        summary_writer = tf.summary.FileWriter(
            opt.checkpoint_dir + '/s%.1d_%.3d/' % (opt.seq_length, opt.img_height) + opt.model_name, sess.graph)
        self.saver = tf.train.Saver()
        # COUNT PARAM
        total_num_parameters = 0
        for variable in tf.trainable_variables():
            total_num_parameters += np.array(variable.get_shape().as_list()).prod()
         print("number of trainable parameters: {}".format(total_num_parameters))
        # INIT
        sess.run(tf.global_variables_initializer())
        sess.run(tf.local_variables_initializer())
        coordinator = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coordinator)
        # LOAD CHECKPOINT IF SET
        if opt.continue_train:
             print("Resume training from previous checkpoint")
             checkpoint = tf.train.latest_checkpoint(
                 os.path.join(opt.checkpoint_dir, 's%.1d_%.3d' % (opt.seq_length, opt.img_height), opt.model_name))
             self.saver.restore(sess, checkpoint)
        if opt.re_train:
            sess.run(global_step.assign(0))

Answer 1

I'm using Jupyter Notebooks to run my models and I recently realize that this error was occuring due to the fact that variables of my model was been saved on an "outter context". So when I restart kernel (and thus clean all my workspace variable) and run all cells the error gone away.

Answer 2

It's because of the for loop in the first part of the code that's possibly missing function name.

The loop tries to create the left_src_tgt_warp/ICSTN/icnv1/weight (same for icnv2 and so on):

def foo(num_layers):
    opt = tf.placeholder(tf.float32, [None, 64])
    for i in range(num_layers):
        with tf.variable_scope("icnv1"):
            feat = tf.layers.dense(opt, units=1, activation=tf.nn.relu)

foo(5)

ValueError: Variable icnv1/dense/kernel already exists, disallowed. Did you mean to set reuse=True

You need distinct names for Variables. One way to achieve this is like this:

def foo(num_layers):
    opt = tf.placeholder(tf.float32, [None, 64])
    for i in range(num_layers):
        with tf.variable_scope("icnv1_layer_{}".format(i)):
            feat = tf.layers.dense(opt, units=1, activation=tf.nn.relu)

we now have distinct names for each, icnv1_layer_1, icnv1_layer_2, etc. Depending on the depth.

Unless of course, you want a shared weights (e.g. it's the same layer, updates as one). In that case just set:

with tf.variable_scope("icnv1", reuse=tf.AUTO_REUSE):