reading NI Daq data in a loop using python and and nidaqmx, fails after first iteration through the loop

Question

I'm trying to capture some accelerometer data and collect some serial data from another device, while changing parameters of the serial device in a loop (wanting to test different settings and record the data for each set of parameters). Everything works fine in the first iteration in the loop, but so far no matter what I've tried, I keep getting a "nidaqmx.errors.DaqReadError: The application is not able to keep up with the hardware acquisition." error during the second loop. Here's a quick break down of what I'm trying to do

1. configure the serial device
2. start accelerometer data collection
3. start serial data collection
4. wait for X amount of time and plot the real-time accelerometer data
5. save the serial and accelerometer data to respective files
6. loop back to step 1

These are the things I've tried to fix the problem

1. initialize the task before the loop and just start and stop it at the beginning end of the loop
2. create and initialize the task at the beginning of the loop, and close the task at the end
3. leave the task running the entire time and just clear my data capture buffer
4. create and initialize the task at the beginning of the loop twice, first with twice my buffer size, and then with just my buffer size , and close the task at the end (i was hoping this might flush something

I only care about the accelerometer data around the time that I'm also capturing the serial data, so I'm fine if I destroy the data outside of these times, if that would make things easier. Below is the code i'm using to do this.

import time
import matplotlib.pyplot as plt
import numpy as np
import nidaqmx
from nidaqmx.stream_readers import AnalogMultiChannelReader
from nidaqmx import constants
import threading
from datetime import datetime
from SerialCaptureFunctions import *  # has functions to configure and capture serial data from the device
import os

parentDir = r'C:\temp'

sampling_freq_in = 51200  # in Hz
buffer_in_size = 10000
bufsize_callback = buffer_in_size
buffer_in_size_cfg = round(buffer_in_size * 1)  # clock configuration
chans_in = 4  # set to number of channels on ni device

channels = ['ai%d' % ch for ch in range(chans_in)] # make list of channel names

# Initialize data placeholders
buffer_in = np.zeros((chans_in, buffer_in_size))

os.chdir(parentDir) # change to parent directory

secondCommPort = 9

refresh_rate_plot = 5  # in Hz
data = np.zeros((chans_in, 1))  # will contain a first column with zeros but that's fine

def cfg_read_task(acquisition, deviceName=None, channels=["ai0", "ai1", "ai2", "ai3"], sensitivity=100.0):  # uses above parameters

    if deviceName is None:
        system = nidaqmx.system.System.local()
        print('searching through connected NI devices searching for USB-9234')
        for device in system.devices:
            # print(f"Device Name: {device.name}, Product Type: {device.product_type}")
            # Check if the device is an NI 9234
            if device.product_type == 'USB-9234':
                deviceName = device.name
                # print('Found NI 9234: %s' % (device_name))

        if deviceName is None:
            print('Failed to find NI 9234 module connected. Please insure this device is connected and try again')

    if deviceName is not None:
        for ch in channels:
            channelName = '%s/%s' % (deviceName, ch)
            acquisition.ai_channels.add_ai_accel_chan(
                channelName,
                sensitivity=sensitivity,
                max_val=5,
                min_val=-5,
                current_excit_val=0.002,
                current_excit_source=constants.ExcitationSource.INTERNAL,
                units=constants.AccelUnits.G,
            )

        acquisition.timing.cfg_samp_clk_timing(rate=sampling_freq_in, sample_mode=constants.AcquisitionType.CONTINUOUS,
                                               samps_per_chan=buffer_in_size_cfg)

def reading_task_callback(task_idx, event_type, num_samples, callback_data):  # bufsize_callback is passed to num_samples
    global data
    global buffer_in
    global SENSITIVITY_V_G
    global BIAS_VDC
    global initDataBuff
    global samplesAvaliable

    if initDataBuff:
        data = np.zeros((chans_in, 1))
        initDataBuff = False
        timeout=0
        print('data buffer cleared')

    else:
        timeout=constants.WAIT_INFINITELY

    if running:
        # It may be wiser to read slightly more than num_samples here, to make sure one does not miss any sample,
        # see: https://documentation.help/NI-DAQmx-Key-Concepts/contCAcqGen.html
        buffer_in = np.zeros((chans_in, num_samples))  # double definition ???
        stream_in.read_many_sample(buffer_in, num_samples, timeout=timeout)

        data = np.append(data, buffer_in, axis=1)  # appends buffered data to total variable data
    return 0  # Absolutely needed for this callback to be well defined (see nidaqmx doc).

# Main loop
running = True
time_start = datetime.now()



timesToLoop = 3

for outputRate, gainsToUse in zip([40, 20,], [ [5, 10], [50, 60]]):
    for i, currOutputGain in enumerate(gainsToUse):

        # update configuration on serial device
        sendSerialConfig(secondCommPort, GainPct=currOutputGain, outputRate=outputRate, )


        captureTime = 20.5

        print('should have accelerometer capture for about %0.1f seconds' % captureTime)
        for loopNum in range(timesToLoop): # loop with current parameters a certain amount of times

            #create and configure the task to capture the accelerometer dat
            task_in = nidaqmx.Task()
            cfg_read_task(task_in)

            stream_in = AnalogMultiChannelReader(task_in.in_stream)
            task_in.register_every_n_samples_acquired_into_buffer_event(bufsize_callback*2, reading_task_callback)

            task_in.start()

            # wait a little to let accelerometer capture a couple seconds of data first before starting serial
            time.sleep(2)

            #start serial capture thread
            serialDataCaptureThread = threading.Thread(target=captureSerialData, args=(4, 9))
            serialDataCaptureThread.start()

            totalTimeRunning = 0
            running = True

            numChannels = 2  # number of accelerometer channels to display

            # Dynamically create subplots based on numChannels
            fig, axes = plt.subplots(numChannels, 1, sharex=True, sharey=False)

            # Ensure axes is always a list (even if numChannels=1)
            if numChannels == 1:
                axes = [axes]

            while running: # plot the accelerometer data in real time
                for i in range(numChannels):  # Only loop through active channels
                    axes[i].clear()
                    axes[i].plot(data[i, -sampling_freq_in * 5:].T)

                # Label and axis formatting
                axes[-1].set_xlabel('time [s]')  # X-label on last axis
                for i in range(numChannels):
                    axes[i].set_ylabel('Acceleration (G\'s)')
                xticks = np.arange(0, data[0, -sampling_freq_in * 5:].size, sampling_freq_in)
                xticklabels = np.arange(0, xticks.size, 1)
                axes[-1].set_xticks(xticks)
                axes[-1].set_xticklabels(xticklabels)

                plt.pause(1 / refresh_rate_plot)
                totalTimeRunning += 1 / refresh_rate_plot

                if totalTimeRunning >= captureTime:
                    running = False # stop capture and plotting

            plt.close()

            task_in.close() # close the task to hopefully clear up any internal buffers

            #save data to csv
            accellFilename = 'currentFileName.csv'
            print('saving accelerometer data to %s.' % accellFilename)

            print('pausing to simulate saving the file for now')
            time.sleep(5)

print('finished...')