InvalidArgumentError: Graph execution error in B3

Question

activity = model.fit(train_gen, epochs=10, # Increase number of epochs if you have sufficient hardware

      validation_data=val_gen,
      
      verbose = 1

) Epoch 1/10 Traceback (most recent call last):

File "C:\Users\BLRCSE~1\AppData\Local\Temp/ipykernel_15312/3305335964.py", line 1, in activity = model.fit(train_gen, epochs=10, # Increase number of epochs if you have sufficient hardware

File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 67, in error_handler raise e.with_traceback(filtered_tb) from None

File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\tensorflow\python\eager\execute.py", line 54, in quick_execute tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,

InvalidArgumentError: Graph execution error:

Detected at node 'gradient_tape/sequential_1/dense_5/MatMul/MatMul' defined at (most recent call last): File "C:\Users\BLRCSE513-WS01\anaconda3\lib\runpy.py", line 197, in _run_module_as_main return _run_code(code, main_globals, None, File "C:\Users\BLRCSE513-WS01\anaconda3\lib\runpy.py", line 87, in run_code exec(code, run_globals) File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\spyder_kernels\console_main.py", line 23, in start.main() File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\spyder_kernels\console\start.py", line 328, in main kernel.start() File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\ipykernel\kernelapp.py", line 677, in start self.io_loop.start() File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\tornado\platform\asyncio.py", line 199, in start self.asyncio_loop.run_forever() File "C:\Users\BLRCSE513-WS01\anaconda3\lib\asyncio\base_events.py", line 596, in run_forever self._run_once() File "C:\Users\BLRCSE513-WS01\anaconda3\lib\asyncio\base_events.py", line 1890, in _run_once handle._run() File "C:\Users\BLRCSE513-WS01\anaconda3\lib\asyncio\events.py", line 80, in _run self._context.run(self._callback, *self._args) File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 457, in dispatch_queue await self.process_one() File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 446, in process_one await dispatch(*args) File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 353, in dispatch_shell await result File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 648, in execute_request reply_content = await reply_content File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\ipykernel\ipkernel.py", line 353, in do_execute res = shell.run_cell(code, store_history=store_history, silent=silent) File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\ipykernel\zmqshell.py", line 533, in run_cell return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs) File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2901, in run_cell result = self._run_cell( File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2947, in _run_cell return runner(coro) File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\IPython\core\async_helpers.py", line 68, in pseudo_sync_runner coro.send(None) File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3172, in run_cell_async has_raised = await self.run_ast_nodes(code_ast.body, cell_name, File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3364, in run_ast_nodes if (await self.run_code(code, result, async=asy)): File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3444, in run_code exec(code_obj, self.user_global_ns, self.user_ns) File "C:\Users\BLRCSE~1\AppData\Local\Temp/ipykernel_15312/1931121224.py", line 1, in activity = model.fit(train_gen, File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 64, in error_handler return fn(*args, **kwargs) File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\engine\training.py", line 1384, in fit tmp_logs = self.train_function(iterator) File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\engine\training.py", line 1021, in train_function return step_function(self, iterator) File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\engine\training.py", line 1010, in step_function outputs = model.distribute_strategy.run(run_step, args=(data,)) File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\engine\training.py", line 1000, in run_step outputs = model.train_step(data) File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\engine\training.py", line 863, in train_step self.optimizer.minimize(loss, self.trainable_variables, tape=tape) File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\optimizer_v2\optimizer_v2.py", line 530, in minimize grads_and_vars = self._compute_gradients( File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\optimizer_v2\optimizer_v2.py", line 583, in _compute_gradients grads_and_vars = self._get_gradients(tape, loss, var_list, grad_loss) File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\optimizer_v2\optimizer_v2.py", line 464, in _get_gradients grads = tape.gradient(loss, var_list, grad_loss) Node: 'gradient_tape/sequential_1/dense_5/MatMul/MatMul' Matrix size-incompatible: In[0]: [32,2], In[1]: [120,1] [[{{node gradient_tape/sequential_1/dense_5/MatMul/MatMul}}]] [Op:__inference_train_function_47374]

Answer 1

If memory growth is enabled for a PhysicalDevice, the runtime initialization will not allocate all memory on the device. Memory growth cannot be configured on a PhysicalDevice with virtual devices configured.

For example:

import tensorflow as tf

physical_devices = tf.config.list_physical_devices('GPU')
try:
  tf.config.experimental.set_memory_growth(physical_devices[0], True)
except:
  # Invalid device or cannot modify virtual devices once initialized.
  pass