Matplotlib define array from text file

Question

So I am trying figure out how to read a text file and plot values from it... I have a text file which is updated every 5 seconds and values are written like this:

"Day, Time, channel1, channel2, channel3, channel4"

Each line is a new 5 second stamp of data.

I want to plot an animated graph of 4 lines (channel1 - channel4) which all share the same x-axis value... how do I define this? below is the pertinent code thus far...

#MATPLOTLIB ANIMATED GRAPH
fig = plt.figure()
ax1 = fig.add_subplot(1,1,1)


ln1, = ax1.plot([], [], 'r-')
ln2, = ax1.plot([], [], 'g-')
ln3, = ax1.plot([], [], 'b-')
ln4, = ax1.plot([], [], 'p-')

def animate(i):
    pullData = open("%s.txt" % FILE_NAME,"r").read()
    dataArray = pullData.split('\n')
    xar = []
    yar = []
    for eachLine in dataArray:
        if len(eachLine)>1:
            x,y = eachLine.split(',')
            ln1.set_data(x1, y1)
            ln2.set_data(x1, y2)
            ln3.set_data(X1, y3)
            ln4.set_data(x1, y4)
    ax1.clear()
    ax1.plot(ln1)
    ax1.plot(ln2)
    ax1.plot(ln3)
    ax1.plot(ln4)
ani = animation.FuncAnimation(fig, animate, interval=5000)
plt.show()

How would I define the x and y for each individual line?

------ Edit #3 -----

import Queue
import datetime as DT
import collections
import matplotlib.pyplot as plt
import numpy as np
import multiprocessing as mp
import time
import matplotlib.dates as mdates
import matplotlib.animation as animation
from ABE_DeltaSigmaPi import DeltaSigma 
from ABE_helpers import ABEHelpers

i2c_helper = ABEHelpers() 
bus = i2c_helper.get_smbus() 
adc = DeltaSigma(bus, 0x68, 0x69, 18)

#Rename file to date
base_dir = '/home/pi/Desktop/DATA'
filename_time = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d')
filename_base = os.path.join(base_dir, filename_time)
filename = '%s.txt' % filename_base

# you will want to change read_delay to 5000
read_delay = int(5000)    # in milliseconds 
write_delay = read_delay/1000.0  # in seconds 
window_size = 60
nlines = 8
datenums = collections.deque(maxlen=window_size)
ys = [collections.deque(maxlen=window_size) for i in range(nlines)]

def animate(i, queue):
    try:
        row = queue.get_nowait()
    except Queue.Empty:
        return
    datenums.append(mdates.date2num(row[0]))
    for i, y in enumerate(row[1:]):
        ys[i].append(y)
    for i, y in enumerate(ys):
        lines[i].set_data(datenums, y)
    ymin = min(min(y) for y in ys)
    ymax = max(max(y) for y in ys)
    xmin = min(datenums)
    xmax = max(datenums)
    if xmin < xmax:
        ax.set_xlim(xmin, xmax)
    ax.set_ylim(ymin, ymax)
    fig.canvas.draw()

def write_data(filename, queue):
    while True:
        delay1 = DT.datetime.now()
        row = []
        for i in range(nlines):
            # read from adc channels and print to screen
            channel = adc.read_voltage(i)
            row.append(channel)

        queue.put([delay1]+row)

        #print voltage variables to local file
        with open(filename, 'a') as DAQrecording:
            time1 = delay1.strftime('%Y-%m-%d')
            time2 = delay1.strftime('%H:%M:%S')
            row = [time1, time2] + row
            row = map(str, row)
            DAQrecording.write('{}\n'.format(', '.join(row)))

        #Delay until next 5 second interval
        delay2 = DT.datetime.now()
        difference = (delay2 - delay1).total_seconds()
        time.sleep(write_delay - difference)

def main():
    global fig, ax, lines
    queue = mp.Queue()
    proc = mp.Process(target=write_data, args=(filename, queue))
    # terminate proc when main process ends
    proc.daemon = True
    # spawn the writer in a separate process
    proc.start()

    fig, ax = plt.subplots()
    xfmt = mdates.DateFormatter('%H:%M:%S')
    ax.xaxis.set_major_formatter(xfmt)
    # make matplotlib treat x-axis as times
    ax.xaxis_date()

    fig.autofmt_xdate(rotation=25)

    lines = []
    for i in range(nlines):
        line, = ax.plot([], [])
        lines.append(line)

    ani = animation.FuncAnimation(fig, animate, interval=read_delay, fargs=(queue,))
    plt.show()    

if __name__ == '__main__':
    main()

Answer 1

I got this to work today, doing everything I need! Thanks for all your help. The final product can:

1. Display readouts from 8 analog sensors in one graph on left, refresh every 5 seconds, and only displays the last 5 minutes of data.
2. The right chart works as a white canvas, displaying the last values which have been read and graphed. I have not been able to figure out how to clear this every 5 seconds, so the text boxes have a white background which covers the old values.
3. All data is written and saved to a text file with date and time stamped with day (D/M/Y) and time (H:M:S).

For anyone else who may be interested in doing something like this, the code is below:

import Queue
import os
import sys
import datetime as DT
import collections
import matplotlib.pyplot as plt
from matplotlib import gridspec
import numpy as np
import multiprocessing as mp
import time
import datetime
import matplotlib.dates as mdates
import matplotlib.animation as animation
from ABE_DeltaSigmaPi import DeltaSigma 
from ABE_helpers import ABEHelpers

i2c_helper = ABEHelpers() 
bus = i2c_helper.get_smbus() 
adc = DeltaSigma(bus, 0x68, 0x69, 16)

#Rename file to date
base_dir = '/home/pi/Desktop/DATA'
ts = time.time()
filename_time = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d')
filename_base = os.path.join(base_dir, filename_time)
filename = '%s.txt' % filename_base

# you will want to change read_delay to 5000
read_delay = int(5000)    # in milliseconds 
write_delay = read_delay/1000.0  # in seconds 
window_size = 60
nlines = 8
ypadding = 0.5
datenums = collections.deque(maxlen=window_size)
ys = [collections.deque(maxlen=window_size) for i in range(nlines)]

def animate(i, queue):
    try:
        row = queue.get_nowait()
    except Queue.Empty:
        return
    datenums.append(mdates.date2num(row[0]))
    for i, y in enumerate(row[1:]):
        ys[i].append(y)
    for i, y in enumerate(ys):
        lines[i].set_data(datenums, y)
    ymin1 = min(min(y) for y in ys)
    ymin = ymin1 - ypadding
    ymax1 = max(max(y) for y in ys)
    ymax = ymax1 + ypadding
    xmin = min(datenums)
    xmax = max(datenums)
    if xmin < xmax:
        ax1.set_xlim(xmin, xmax)
    ax1.set_ylim(ymin, ymax)

    ax2.plot(0, 0)
    ax2.set_xlim(0, 1)
    ax2.set_ylim(0, 1)
    channel1 = row[-8]
    channel2 = row[-7]
    channel3 = row[-6]
    channel4 = row[-5]
    channel5 = row[-4]
    channel6 = row[-3]
    channel7 = row[-2]
    channel8 = row[-1]

    ax2.text(0.1,0.8,'CH1: %.02f \n CH2: %.02f \n CH3: %.02f \n CH4: %.02f \n CH5: %.02f \n CH6: %.02f \n CH7: %.02f \n CH8: %.02f \n' % (channel1,channel2,channel3,channel4,channel5,channel6,channel7,channel8) , ha='left', va='top', backgroundcolor='w')
    fig.canvas.draw()

def write_data(filename, queue):
    while True:
        delay1 = DT.datetime.now()
        row = []
        for i in range(nlines):
            # read from adc channels and print to screen
            channel = adc.read_voltage(i)
            row.append(channel)

        queue.put([delay1]+row)

        #print voltage variables to local file
        with open(filename, 'a') as DAQrecording:
            time1 = delay1.strftime('%Y-%m-%d')
            time2 = delay1.strftime('%H:%M:%S')
            row = [time1, time2] + row
            row = map(str, row)
            DAQrecording.write('{}\n'.format(', '.join(row)))

        #Delay until next 5 second interval
        delay2 = DT.datetime.now()
        difference = (delay2 - delay1).total_seconds()
        time.sleep(write_delay - difference)

def main():
    global fig, ax1, ax2, lines
    queue = mp.Queue()
    proc = mp.Process(target=write_data, args=(filename, queue))
    # terminate proc when main process ends
    proc.daemon = True
    # spawn the writer in a separate process
    proc.start()

    fig, (ax1, ax2) = plt.subplots(1, 2, sharey=False) 
    gs = gridspec.GridSpec(1,2, width_ratios=[3, 1] wspace=None)
    ax1 = plt.subplot(gs[0])
    ax2 = plt.subplot(gs[1])
    ax2.axes.xaxis.set_ticklabels([])
    ax2.axes.yaxis.set_ticklabels([])
    xfmt = mdates.DateFormatter('%H:%M:%S')
    ax1.xaxis.set_major_formatter(xfmt)

    # make matplotlib treat x-axis as times
    ax1.xaxis_date()
    fig.autofmt_xdate()
    fig.suptitle('Data Acquisition', fontsize=14, fontweight='bold')

    lines = []
    for i in range(nlines):
        line, = ax1.plot([], [])
        lines.append(line)

    ani = animation.FuncAnimation(fig, animate, interval=read_delay, fargs=(queue,))
    plt.show()    

if __name__ == '__main__':
    main()

Answer 2

Writing and reading from the same file would require a lock to prevent a race condition -- reading from a file before it is fully written. It's possible, but below I suggest a different way.

Since both programs are written in Python, you could use the multiprocessing module to spawn the writer process, and have it write values to a Queue. Then the main process can have animate get values from the Queue and draw the result. Queue handles the locking and interprocess communication for us, and allows us to transfer the datetime object and float values as Python objects without having to read them from a file and parse the strings.

import Queue
import datetime as DT
import collections
import matplotlib.pyplot as plt
import numpy as np
import multiprocessing as mp
import time
import matplotlib.dates as mdates
import matplotlib.animation as animation
try:
    from ABE_DeltaSigmaPi import DeltaSigma 
    from ABE_helpers import ABEHelpers
    i2c_helper = ABEHelpers() 
    bus = i2c_helper.get_smbus() 
    adc = DeltaSigma(bus, 0x68, 0x69, 18)
except ImportError:
    class ADC(object):
        """
        This is a dummy class to mock the adc.read_voltage calls.
        """
        def __init__(self):
            self.x = 0
        def read_voltage(self, i):
            if i == 0:
                self.x += 0.1
            return np.sin(self.x/10)*(i+1)

    adc = ADC()

filename = 'data.txt'
# you will want to change read_delay to 5000
read_delay = int(0.05 * 1000)    # in milliseconds 
write_delay = read_delay/1000.0  # in seconds 
window_size = 60
nlines = 8
datenums = collections.deque(maxlen=window_size)
ys = [collections.deque(maxlen=window_size) for i in range(nlines)]

def animate(i, queue):
    try:
        row = queue.get_nowait()
    except Queue.Empty:
        return
    datenums.append(mdates.date2num(row[0]))
    for i, y in enumerate(row[1:]):
        ys[i].append(y)
    for i, y in enumerate(ys):
        lines[i].set_data(datenums, y)
    ymin = min(min(y) for y in ys)
    ymax = max(max(y) for y in ys)
    xmin = min(datenums)
    xmax = max(datenums)
    if xmin < xmax:
        ax.set_xlim(xmin, xmax)
    ax.set_ylim(ymin, ymax)
    fig.canvas.draw()

def write_data(filename, queue):
    while True:
        delay1 = DT.datetime.now()
        row = []
        for i in range(nlines):
            # read from adc channels and print to screen
            channel = adc.read_voltage(i)
            temp = 3.45 * channel
            row.append(temp)

        queue.put([delay1]+row)

        #print voltage variables to local file
        with open(filename, 'a') as DAQrecording:
            time1 = delay1.strftime('%Y-%m-%d')
            time2 = delay1.strftime('%H:%M:%S')
            row = [time1, time2] + row
            row = map(str, row)
            DAQrecording.write('{}\n'.format(', '.join(row)))

        #Delay until next 5 second interval
        delay2 = DT.datetime.now()
        difference = (delay2 - delay1).total_seconds()
        time.sleep(write_delay - difference)

def main():
    global fig, ax, lines
    queue = mp.Queue()
    proc = mp.Process(target=write_data, args=(filename, queue))
    # terminate proc when main process ends
    proc.daemon = True
    # spawn the writer in a separate process
    proc.start()

    fig, ax = plt.subplots()
    xfmt = mdates.DateFormatter('%H:%M:%S')
    ax.xaxis.set_major_formatter(xfmt)
    # make matplotlib treat x-axis as times
    ax.xaxis_date()

    fig.autofmt_xdate(rotation=25)

    lines = []
    for i in range(nlines):
        line, = ax.plot([], [])
        lines.append(line)

    ani = animation.FuncAnimation(fig, animate, interval=read_delay, fargs=(queue,))
    plt.show()    

if __name__ == '__main__':
    main()