Python data display with graph

Question

I am just dipping my toes into Python and have had help on here to get a live updating Matplotlib graph to work for me. This program uses animation to pull data from a dynamically growing CSV file with data that looks like the following:

TimeStamp, ReadCount, Antenna, Protocol, RSSI, EPC, Sensor
09/28/2016 17:34:28.967, 5686, 2, GEN2, -25, E036115348A908CB, 23.16,0.00,0.00, (Infinity%), 
09/28/2016 17:34:29.716, 5687, 2, GEN2, -32, E036115348A908CB,  (Infinity%), 
09/28/2016 17:34:31.155, 5689, 2, GEN2, -27, E036115348A908CB, 22.74,3.38, (Infinity%), 
09/28/2016 17:34:32.351, 5692, 2, GEN2, -25, E036115348A908CB, 22.95,0.00,0.00,3.38, (Infinity%), 
09/28/2016 17:34:32.895, 5695, 2, GEN2, -23, E036115348A908CB, 22.95,0.00,0.00,3.38, (Infinity%), 
09/28/2016 17:34:33.397, 5698, 2, GEN2, -21, E036115348A908CB, 23.78,0.00,0.00,3.38, (Infinity%), 
09/28/2016 17:34:33.946, 5699, 2, GEN2, -23, E036115348A908CB, 23.57,0.00,3.38, (Infinity%), 
09/28/2016 17:34:34.912, 5702, 2, GEN2, -27, E036115348A908CB, 23.36,0.00,0.00,3.38, (Infinity%), 
09/28/2016 17:34:35.394, 5705, 2, GEN2, -25, E036115348A908CB, 23.36,0.00,0.00,3.38, (Infinity%), 
09/28/2016 17:34:35.988, 5707, 2, GEN2, -23, E036115348A908CB, 23.78,0.00,0.00,3.38, (Infinity%), 
09/28/2016 17:34:36.489, 5710, 2, GEN2, -21, E036115348A908CB, 23.99,0.00,0.00,3.38, (Infinity%), 
09/28/2016 17:34:37.269, 5712, 2, GEN2, -23, E036115348A908CB, 23.78,0.00,0.00,3.38, (Infinity%), 
09/28/2016 17:34:37.796, 5715, 2, GEN2, -18, E036115348A908CB, 23.78,0.00,0.00,3.38, (Infinity%), 
09/28/2016 17:34:38.296, 5718, 2, GEN2, -7, E036115348A908CB, 22.32,0.00,0.00,3.38, (Infinity%), 
09/28/2016 17:34:38.826, 5721, 2, GEN2, -7, E036115348A908CB, 23.57,0.00,0.00,3.38, (Infinity%), 
09/28/2016 17:34:39.320, 5724, 2, GEN2, -30, E036115348A908CB, 23.36,0.00,0.00,3.38, (Infinity%), 
09/28/2016 17:34:39.870, 5727, 2, GEN2, -9, E036115348A908CB, 23.36,0.00,0.00,3.38, (Infinity%), 

This data is coming in from a sensor, and I'd like to be able to pull/display some other values which this data contains on a display along with this graph. The code for the plot looks like the following:

import matplotlib
import matplotlib.pyplot as plt
import matplotlib.animation as animation

from datetime import datetime, timedelta
import collections
import csv

offset = 16
slope = -.2081

def plot(ax, data, colour, width):
    if data:
        last_dt = data[0][0]
        gap = timedelta(seconds=10)

        x = []
        y = []

        # Plot groups of data not more than 60 seconds apart
        for dt, ten in data:
            if dt <= last_dt + gap:
                x.append(dt)
                y.append(ten)
            else:
                ax.plot(matplotlib.dates.date2num(x), y, colour, linewidth=width)
                x = [dt]
                y = [ten]

            last_dt = dt

        ax.plot(matplotlib.dates.date2num(x), y, colour, linewidth=width)


def animate(i, fig, ax):
    # Read in the CSV file
    data = collections.defaultdict(list)
    fields = ["TimeStamp", "ReadCount", "Antenna", "Protocol", "RSSI", "EPC", "Temp", "Ten", "Powr", "Unpowr", "Inf"]

    with open('SensorLogFile.csv') as f_input:
        csv_input = csv.DictReader(f_input, skipinitialspace=True, fieldnames=fields)
        header = next(csv_input)

        # Separate the rows based on the Antenna field
        for row in csv_input:
            try:
                data[row['Antenna']].append(
                    [datetime.strptime(row['TimeStamp'], '%m/%d/%Y %H:%M:%S.%f'), 
                    int(float(row['Ten']) * float(slope) + float(offset))])
            except:
                pass

    # Drop any data points more than 1.5 mins older than the last entry

    latest_dt = data[row['Antenna']][-1][0]     # Last entry
    not_before = latest_dt - timedelta(minutes=.25)

    for antenna, entries in data.items():
        data[antenna] = [[dt, count] for dt, count in entries if dt >= not_before]

    # Redraw existing axis
    ax.clear()

    ax.spines['bottom'].set_color("#5998ff")
    ax.spines['top'].set_color("#5998ff")
    ax.spines['left'].set_color("#5998ff")
    ax.spines['right'].set_color("#5998ff")

    hfmt = matplotlib.dates.DateFormatter('%m/%d/%Y\n%I:%M:%S %p')
    ax.xaxis.set_major_formatter(hfmt)
    fig.autofmt_xdate()

    plot(ax, data['1'], 'c', 6)     # Antenna 1
    plot(ax, data['2'], 'r', 6)     # Antenna 2
    plot(ax, data['3'], 'y', 6)     # Antenna 3

    ax.grid(True, color='w')
    plt.ylabel('Tension (lb)', color='w', fontsize=20)
    plt.title('Spiral 1 Tension', color='w', fontsize=26)

    ax.tick_params(axis='y', colors='w')
    ax.tick_params(axis='x', colors='w')

#    ax.set_ylim([21,28])

fig = plt.figure(facecolor='#07000d')
ax = fig.add_subplot(111, axisbg='#07000d')

ani = animation.FuncAnimation(fig, animate, fargs=(fig, ax), interval=250)
plt.show()

I'd like to be able to have the graph on top and an open area below the graph (the full width of the graph) where I could put the following (probably have a box for each of these pieces of data):

• "Temp" (7th csv on each line, just need to convert to deg F)
• "Time" (current time)
• "Overdrive" (not showing anything with this data but just a placeholder for it in the future)
• "Cycles" (would be the number of times that Antenna 1 is first read after a minute gap)
• box with an image on the bottom right - for the example lets say this image

I have looked into using Tkinter and tried to follow sentdex's video here

But have not been able to get it to work.

I am on a time crunch and can't struggle solo on this anymore - need some help from the pros here. Can anyone give me some direction/guidance on how to make a display like what I'm looking for?

Answer 1

To get you started, there are probably some questions to be answered first.

1. How live is live? Should it update in real time? In this case the use of FuncAnimation is probably a bad idea. Answer: yes, real time
2. How fast does it have to be, that is, how many updates per second? Answer: Update 4 times per second
3. Is there a trigger for the updates? Do you know when new data arrives? Answer: No real trigger, but see 2.
4. How much interactivity do you need? Is it just the plot that should open up and then run for itself or do you need interactive buttons/sliders that allow you to select part of the data? Even with some basic interactivity I would suggest to stay in matplotlib with an interactive backend and not fiddle around with tkinter too much - given that you seem to know nothing about it and are limited in time. Answer: No interactivity needed

5. When it comes to your data, I see a problem. Looking at those lines
   E036115348A908CB, 22.74,3.38, (Infinity%),
E036115348A908CB, 22.95,0.00,0.00,3.38, (Infinity%),
   there seem to be cases where the data is not complete?! Is this avoidable?

6. What about the time axis for the plot? First, you #Drop any data points more than 1.5 mins older than the last entry and then you #Plot groups of data not more than 60 seconds apart , and then you discard any data which is older then gap = timedelta(seconds=10). Can you tell us what "groups of data" means and how long the time axis should be?

7. How would you calculate the number of cycles?

---

Result (so far): Here we go, this is what you can easily get with matplotlib.

import matplotlib
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

from datetime import datetime, timedelta
import collections
import csv

class Anim():
    """
    This class provides a "live" plot of the contents of a log file in csv format. 
    The class structure makes it easy to separate the plot generation from the
        frequent updating of the plot.
    The code is based on a question at stackoverflow
        http://stackoverflow.com/questions/39858501/python-data-display-with-graph
    """
    def __init__(self):

        self.offset = 16.
        self.slope = -.2081

        self.i = 0

        self.axisbg = '#07000d'

        self.fig = plt.figure(figsize=(15,8), facecolor=self.axisbg)
        self.ax = self.fig.add_subplot(111, axisbg=self.axisbg)

        [self.ax.spines[wh].set_color("#5998ff") for wh in ['bottom', 'top', 'left', 'right']]

        self.hfmt = matplotlib.dates.DateFormatter('%m/%d/%Y\n%I:%M:%S %p')
        self.ax.xaxis.set_major_formatter(self.hfmt)
        self.fig.autofmt_xdate()

        #self.framenumber = plt.figtext(0.9, .9, "0",  color='w')
        self.ax.grid(True, color='w')
        plt.ylabel('Tension (lb)', color='w', fontsize=20)
        plt.title('Spiral 1 Tension', color='w', fontsize=26)

        self.ax.tick_params(axis='y', colors='w')
        self.ax.tick_params(axis='x', colors='w')

        initialx = [self.stime("09/28/2016 17:34:28.967"),self.stime("09/28/2016 17:34:29.716") ]
        initialy = [0,0 ]

        self.line1, = self.ax.plot(matplotlib.dates.date2num(initialx), initialy, color="c", linewidth=6)
        self.line2, = self.ax.plot(matplotlib.dates.date2num(initialx), initialy, color="r", linewidth=6)
        self.line3, = self.ax.plot(matplotlib.dates.date2num(initialx), initialy, color="y", linewidth=6)

        plt.subplots_adjust(left=0.1, bottom=0.28, right=0.9, top=0.9, wspace=0, hspace=0)

        self.ax_temp =      plt.axes([0.1, 0.08, 0.2, 0.06], axisbg=self.axisbg)
        self.ax_time =      plt.axes([0.2, 0.08, 0.2, 0.06], axisbg=self.axisbg)
        self.ax_overdrive = plt.axes([0.4, 0.08, 0.2, 0.06], axisbg=self.axisbg)
        self.ax_cycles =    plt.axes([0.5, 0.08, 0.2, 0.06], axisbg=self.axisbg)
        self.ax_image =    plt.axes([0.75, 0.03, 0.3, 0.2], axisbg=self.axisbg)

        self.tx_temp = self.ax_temp.text(0,0, "Temp", color="w", transform=self.ax_temp.transAxes, bbox={"pad" : 10, "ec" : "w", "fc" : self.axisbg})
        self.tx_time = self.ax_time.text(0,0, "Time", color="w", transform=self.ax_time.transAxes, bbox={"pad" : 10, "ec" : "w", "fc" : self.axisbg})
        self.tx_overdrive = self.ax_overdrive.text(0,0, "Overdrive", color="w", transform=self.ax_overdrive.transAxes, bbox={"pad" : 10, "ec" : "w", "fc" : self.axisbg})
        self.tx_cycles = self.ax_cycles.text(0,0, "Cyles", color="w", transform=self.ax_cycles.transAxes, bbox={"pad" : 10, "ec" : "w", "fc" : self.axisbg})

        self.ax_image.imshow(mpimg.imread('mmbRy.jpg'))
        self.ax_image.tick_params(axis='x',which='both',bottom='off', top='off',labelbottom='off')
        self.ax_image.tick_params(axis='y',which='both',left='off', right='off',labelleft='off')
        [self.ax_image.spines[wh].set_color("#5998ff") for wh in ['bottom', 'top', 'left', 'right']]

        self.timer = self.fig.canvas.new_timer(interval=250, callbacks=[(self.animate, [], {})])
        self.timer.start()
        plt.show()


    def plot(self, data, line):
        if data:
            last_dt = data[0][0]
            gap = timedelta(seconds=10)
            x = []
            y = []
            # Plot groups of data not more than 60 seconds apart
            for dt, ten in data:
                if dt <= last_dt + gap:
                    x.append(dt)
                    y.append(ten)
                else:
                    line.set_data(matplotlib.dates.date2num(x), y)
                    #ax.plot(, colour, linewidth=width)
                    x = [dt]
                    y = [ten]
                last_dt = dt
            line.set_data(matplotlib.dates.date2num(x), y)


    def animate(self):
        self.i +=1 #counting the number of frames
        # Read in the CSV file
        data = collections.defaultdict(list)
        fields = ["TimeStamp", "ReadCount", "Antenna", "Protocol", "RSSI", "EPC", "Temp", "Ten", "Powr", "Unpowr", "Inf"]
        temp = ""
        # the complete file is read in, which might be a problem once the file gets very large
        with open('SensorLogFile.csv') as f_input:
            csv_input = csv.DictReader(f_input, skipinitialspace=True, fieldnames=fields)
            header = next(csv_input)
            # Separate the rows based on the Antenna field
            for row in csv_input:
                try:
                    data[row['Antenna']].append([self.stime(row['TimeStamp']), self.rten(row['Ten']) ])
                    temp= row['Temp']
                except:
                    pass

        # Drop any data points more than 1.5 mins older than the last entry
        latest_dt = data[row['Antenna']][-1][0]     # Last entry
        not_before = latest_dt - timedelta(minutes=.25)

        for antenna, entries in data.items():
            data[antenna] = [[dt, count] for dt, count in entries if dt >= not_before]

        self.plot(data['1'],  self.line1)     # Antenna 1
        self.plot(data['2'],  self.line2)     # Antenna 2
        self.plot(data['3'],  self.line3)     # Antenna 3

        #Filling the text boxes
        self.tx_temp.set_text(u"Temperature\n{temp:.2f} °F".format(temp=self.deg2F(temp)))
        self.tx_time.set_text("Time\n{time}".format(time=datetime.now().time()) )
        self.tx_overdrive.set_text("Overdrive\nfill later")
        #Todo: how do you calculate this?
        self.tx_cycles.set_text("Cyles\n{cyles}".format(cyles=self.i)) 
        #Todo: setting the limits correctly, depending on the user's need
        self.ax.set_ylim([0,16])     
        self.ax.set_xlim([matplotlib.dates.date2num(not_before), matplotlib.dates.date2num(latest_dt)])
        #Update the canvas
        self.fig.canvas.draw()


    def deg2F(self,deg):
        return float(deg) * 9./5. + 32. 

    def stime(self, timestamp):
        return datetime.strptime(timestamp, '%m/%d/%Y %H:%M:%S.%f')

    def rten(self, ten):
        return int(float(ten) * self.slope + self.offset)


if __name__ == "__main__":
    Anim()