Python: Failed to convert a NumPy array to a Tensor

Question

I am sorry that this question has been asked many times, but after reading all the posts, I still cannot tell what is wrong with my code. I feel very lost now and I would be super greatful if anyone can help!

According to the answer of this post, it might because my X and Y having different length. Also, my process() function triggers the error message below. I have tried to print the length of X and Y, which were equal.

:61: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray. X = np.array(padded_encoded_essays)

Sample data

train_data = [   {'corrected': 'have a good day', 'father': 1},
         {'corrected': 'i suggest you see this movie', 'father': 1},
         {'corrected': 'The afternoon grew so glowering that in the sixth inning the arc lights were turned on--always a wan sight in the daytime, like the burning headlights of a funeral procession. Aided by the gloom, Fisher was slicing through the Sox rookies, and Williams did not come to bat in the seventh. He was second up in the eighth. This was almost certainly his last time to come to the plate in Fenway Park, and instead of merely cheering, as we had at his three previous appearances, we stood, all of us, and applauded.', 'father': 2},
         {'corrected': 'worse than any show', 'father': 1},
         {'corrected': 'nice movie, so love it', 'father': 2},
         {'corrected': "The day I picked my dog up from the pound was one of the happiest days of both of our lives. I had gone to the pound just a week earlier with the idea that I would just 'look' at a puppy. Of course, you can no more just look at those squiggling little faces so filled with hope and joy than you can stop the sun from setting in the evening. I knew within minutes of walking in the door that I would get a puppy… but it wasn't until I saw him that I knew I had found my puppy", 'father': 2}
    ]

train_data= pd.DataFrame(train_data)

My code

embed = hub.load("https://tfhub.dev/google/Wiki-words-250/2")

def get_word_count(essay):
    """
    get the number of vocab in the essay
    """
    return len(essay)

def get_word2vec_enc(essays):
    """
    get word2vec value for each word in sentence.
    concatenate word in numpy array, so we can use it as RNN input
    """
    encoded = []
    for essay in essays:
        tokens = essay.split(" ")
        word2vec_embedding = embed(tokens)
        encoded.append(word2vec_embedding)
    return encoded

def get_padded_encoded_essays(encoded_essays):
    """
    for short essays, we prepend zero padding so all input to RNN has same length,
    for long essays, we truncate it to the first 250 words
    """
    padded_essays_encoding = []
    for enc_essay in encoded_essays:
        if get_word_count(enc_essay)> 250:
            enc_essay[:249]
        
        else:
            zero_padding_cnt = 250 - enc_essay.shape[0]
            pad = np.zeros((1, 250))
            for i in range(zero_padding_cnt):
                enc_essay = np.concatenate((pad, enc_essay), axis=0)
        padded_essays_encoding.append(enc_essay)
    return padded_essays_encoding

def ses_encode(ses):
    """
    return one hot encoding for Y value
    """
    if ses == 1: 
        return [1,0]  # for high ses
    else: 
        return [0,1]  # for low ses
    
def preprocess(df):
    """
    encode text value to numeric value
    """
 
    # encode words into word2vec
    essays = df['corrected'].tolist()

    
    encoded_essays = get_word2vec_enc(essays)
    padded_encoded_essays = get_padded_encoded_essays(encoded_essays)
    # encoded ses
    sess = df['father'].tolist()

    encoded_ses = [ses_encode(ses) for ses in sess]
    X = np.array(padded_encoded_essays)
    Y = np.array(encoded_ses)
    
#     tf.convert_to_tensor(X)
#     tf.convert_to_tensor(Y)

    return X, Y


# LSTM model
model = Sequential()
model.add(LSTM(32))
model.add(Dense(2, activation='softmax'))


model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

# Training 
print('Train...')
model.fit(train_X, train_Y,epochs=50)

Stack track

Train...
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-91-932bd9f83059> in <module>
      1 print('Train...')
----> 2 model.fit(train_X, train_Y,epochs=50)

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\keras\engine\training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
   1131          training_utils.RespectCompiledTrainableState(self):
   1132       # Creates a `tf.data.Dataset` and handles batch and epoch iteration.
-> 1133       data_handler = data_adapter.get_data_handler(
   1134           x=x,
   1135           y=y,

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py in get_data_handler(*args, **kwargs)
   1362   if getattr(kwargs["model"], "_cluster_coordinator", None):
   1363     return _ClusterCoordinatorDataHandler(*args, **kwargs)
-> 1364   return DataHandler(*args, **kwargs)
   1365 
   1366 

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py in __init__(self, x, y, sample_weight, batch_size, steps_per_epoch, initial_epoch, epochs, shuffle, class_weight, max_queue_size, workers, use_multiprocessing, model, steps_per_execution, distribute)
   1152     adapter_cls = select_data_adapter(x, y)
   1153     self._verify_data_adapter_compatibility(adapter_cls)
-> 1154     self._adapter = adapter_cls(
   1155         x,
   1156         y,

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py in __init__(self, x, y, sample_weights, sample_weight_modes, batch_size, epochs, steps, shuffle, **kwargs)
    245                **kwargs):
    246     super(TensorLikeDataAdapter, self).__init__(x, y, **kwargs)
--> 247     x, y, sample_weights = _process_tensorlike((x, y, sample_weights))
    248     sample_weight_modes = broadcast_sample_weight_modes(
    249         sample_weights, sample_weight_modes)

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py in _process_tensorlike(inputs)
   1044     return x
   1045 
-> 1046   inputs = nest.map_structure(_convert_numpy_and_scipy, inputs)
   1047   return nest.list_to_tuple(inputs)
   1048 

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\util\nest.py in map_structure(func, *structure, **kwargs)
    865 
    866   return pack_sequence_as(
--> 867       structure[0], [func(*x) for x in entries],
    868       expand_composites=expand_composites)
    869 

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\util\nest.py in <listcomp>(.0)
    865 
    866   return pack_sequence_as(
--> 867       structure[0], [func(*x) for x in entries],
    868       expand_composites=expand_composites)
    869 

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py in _convert_numpy_and_scipy(x)
   1039       if issubclass(x.dtype.type, np.floating):
   1040         dtype = backend.floatx()
-> 1041       return ops.convert_to_tensor_v2_with_dispatch(x, dtype=dtype)
   1042     elif _is_scipy_sparse(x):
   1043       return _scipy_sparse_to_sparse_tensor(x)

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\util\dispatch.py in wrapper(*args, **kwargs)
    204     """Call target, and fall back on dispatchers if there is a TypeError."""
    205     try:
--> 206       return target(*args, **kwargs)
    207     except (TypeError, ValueError):
    208       # Note: convert_to_eager_tensor currently raises a ValueError, not a

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\framework\ops.py in convert_to_tensor_v2_with_dispatch(value, dtype, dtype_hint, name)
   1428     ValueError: If the `value` is a tensor not of given `dtype` in graph mode.
   1429   """
-> 1430   return convert_to_tensor_v2(
   1431       value, dtype=dtype, dtype_hint=dtype_hint, name=name)
   1432 

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\framework\ops.py in convert_to_tensor_v2(value, dtype, dtype_hint, name)
   1434 def convert_to_tensor_v2(value, dtype=None, dtype_hint=None, name=None):
   1435   """Converts the given `value` to a `Tensor`."""
-> 1436   return convert_to_tensor(
   1437       value=value,
   1438       dtype=dtype,

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\profiler\trace.py in wrapped(*args, **kwargs)
    161         with Trace(trace_name, **trace_kwargs):
    162           return func(*args, **kwargs)
--> 163       return func(*args, **kwargs)
    164 
    165     return wrapped

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\framework\ops.py in convert_to_tensor(value, dtype, name, as_ref, preferred_dtype, dtype_hint, ctx, accepted_result_types)
   1564 
   1565     if ret is None:
-> 1566       ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
   1567 
   1568     if ret is NotImplemented:

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\framework\tensor_conversion_registry.py in _default_conversion_function(***failed resolving arguments***)
     50 def _default_conversion_function(value, dtype, name, as_ref):
     51   del as_ref  # Unused.
---> 52   return constant_op.constant(value, dtype, name=name)
     53 
     54 

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\framework\constant_op.py in constant(value, dtype, shape, name)
    262     ValueError: if called on a symbolic tensor.
    263   """
--> 264   return _constant_impl(value, dtype, shape, name, verify_shape=False,
    265                         allow_broadcast=True)
    266 

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\framework\constant_op.py in _constant_impl(value, dtype, shape, name, verify_shape, allow_broadcast)
    274       with trace.Trace("tf.constant"):
    275         return _constant_eager_impl(ctx, value, dtype, shape, verify_shape)
--> 276     return _constant_eager_impl(ctx, value, dtype, shape, verify_shape)
    277 
    278   g = ops.get_default_graph()

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\framework\constant_op.py in _constant_eager_impl(ctx, value, dtype, shape, verify_shape)
    299 def _constant_eager_impl(ctx, value, dtype, shape, verify_shape):
    300   """Implementation of eager constant."""
--> 301   t = convert_to_eager_tensor(value, ctx, dtype)
    302   if shape is None:
    303     return t

~\anaconda3\envs\tf\lib\site-packages\tensorflow\python\framework\constant_op.py in convert_to_eager_tensor(value, ctx, dtype)
     96       dtype = dtypes.as_dtype(dtype).as_datatype_enum
     97   ctx.ensure_initialized()
---> 98   return ops.EagerTensor(value, ctx.device_name, dtype)
     99 
    100 

ValueError: Failed to convert a NumPy array to a Tensor (Unsupported object type tensorflow.python.framework.ops.EagerTensor).

Thank you in advance!

Answer 1

For community benefit adding @Yatharth Malik answer here,

Try np.vstack instead of using np.array, as the former converts data into 2D matrix while latter is nested arrays X = np.vstack(padded_encoded_essays)

Y = np.vstack(encoded_ses)