How to check limit range from csv file in python script

Question

I am trying to fetch some values from database and need to check some lower and upper limits of a variables which are store in a text file like this and they are separated by \t. the text file looks like

Variable      lower_limit   upper_limit
temperature   20            40
pressure      0             100
temperature2  0             30
temperature3  20            25

and the data in database looks like

usec        temperature_data    temperature2_data
1456411800  25                  15
1456412400  45                  25
1456413000  28                  19

So i start with checking first whether the variable is in the text file, if yes then i would need to check the limits of that variable. until now i am only successful in verifying the name of the variable, but i am unable to check the limits. my code is as follow

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import datetime as dt
import sys
import time
import datetime
import calendar
import numpy as np
import mysql.connector
import datetime
import numpy as np
import pandas as pd
import mysql.connector
from mysql.connector import errorcode


# starting day, month and year
start_day = dt.datetime(2016, 02, 25)
# total number of dates to visualize
number = 11
num_total = 11
# enter limit range
upper_limit = 250 # these are hardcode values which i want to 
                    replace and instead of hard code , i want to
                    check these limits values from the text file
lower_limit = 0
# start day in epoch time format
start_time = 1456411800

# variable name and filepath
filepath = '/home/robbyy/files/limit.txt'
vari_name = 'temperature2'

# database name, user and password details and query to fetch   respective data
usr = 'roby'
password = 'xxxx'
db_name = 'roby_data'
insert_query = ("SELECT usec , temperature2_data "
            "FROM rob_table WHERE usec >= %s "
            "AND usec <= %s")


def generate_data():
    num = num_total
    cnx = mysql.connector.connect(user=usr, password=password,
                              database=db_name)
    cursor = cnx.cursor()
    query = insert_query

    for i in range(number):
        current_start_ts = (start_time + (i*86400))
        current_day =   datetime.datetime.fromtimestamp(current_start_ts)
        # print 'cd: ', current_day
        current_end_ts = (start_time + (i*86400)) + 86399
        cursor.execute(query, (current_start_ts * 1000000,
                           current_end_ts * 1000000))
        rows = cursor.fetchall()
        rows_arr = np.array(rows)
        # print 'rows all here li: ', rows
        with open(filepath, 'r') as f:
            limit_file = f.read()
        limits = {}
        for line in limit_file.splitlines():
            print 'line to see:', line
            variable, lower, upper = line.split()
            if not variable == 'Variable':
                limits[variable] = {'lower': int(lower),
                                    'upper': int(upper)}
        print 'limits: ', limits
        if vari_name in data:
            pass
                if len(rows_arr) == 0:
                    continue
                    # print 'no data is here'
                else:
                    for item, index in rows_arr:
                        if index >= upper_limit or index <= lower_limit:
                            print 'data exceeds limit: ', index
                        else:
                            continue
                            # print 'data is within range: ', index
        else:
            print 'sorry: this variable name is invalid'

    start = start_day
    dates = [start + dt.timedelta(days=i) for i in range(num)]
    return dates


def main():
    dates = generate_data()


main()

If someone helps me or guide me how to check the lower and upper limits from the text file for the required variable instead of giving hard coded values in the script. i would be grateful thanks

Answer 1

just parse the limits file and for example create a dict out of it. Something like this.

def parse_limits(file):
    with open(file, 'r') as f:
       limit_file = f.read()
    limits = {}
    for line in limit_file.splitlines():
        variable, lower, upper = line.split()
        if not variable == 'Variable':
            limits[variable] = {'lower': int(lower),
                                'upper': int(upper)}
    return limits

That would result in a nested dict as follows:

{
    'pressure': {'upper': 100, 'lower': 0}, 
    'temperature2': {'upper': 30, 'lower': 0}, 
    'temperature': {'upper': 40, 'lower': 20}, 
    'temperature3': {'upper': 25, 'lower': 20}
}

Edit:

As requested your final code might look s.th. like this:

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import datetime as dt
import sys
import time
import datetime
import calendar
import numpy as np
import mysql.connector
import datetime
import numpy as np
import pandas as pd
import mysql.connector
from mysql.connector import errorcode


# starting day, month and year
start_day = dt.datetime(2016, 02, 25)
# total number of dates to visualize
number = 11
num_total = 11
# enter limit range
upper_limit = 250 # these are hardcode values which i want to
                    replace and instead of hard code , i want to
                    check these limits values from the text file
lower_limit = 0
# start day in epoch time format
start_time = 1456411800

# variable name and filepath
filepath = '/home/robbyy/files/limit.txt'
vari_name = 'temperature2'

# database name, user and password details and query to fetch   respective data
usr = 'roby'
password = 'xxxx'
db_name = 'roby_data'
insert_query = ("SELECT usec , temperature2_data "
            "FROM rob_table WHERE usec >= %s "
            "AND usec <= %s")


def parse_limits(file):
    with open(file, 'r') as f:
       limit_file = f.read()
    limits = {}
    for line in limit_file.splitlines():
        variable, lower, upper = line.split()
        if not variable == 'Variable':
            limits[variable] = {'lower': int(lower),
                                'upper': int(upper)}
    return limits


limits = parse_limits(filepath)

def generate_data():
    num = num_total
    cnx = mysql.connector.connect(user=usr, password=password,
                              database=db_name)
    cursor = cnx.cursor()
    query = insert_query

    for i in range(number):
        current_start_ts = (start_time + (i*86400))
        current_day =   datetime.datetime.fromtimestamp(current_start_ts)
        # print 'cd: ', current_day
        current_end_ts = (start_time + (i*86400)) + 86399
        cursor.execute(query, (current_start_ts * 1000000,
                           current_end_ts * 1000000))
        rows = cursor.fetchall()
        rows_arr = np.array(rows)
        # print 'rows all here li: ', rows
        print 'limits: ', limits
        if vari_name in data:
                if len(rows_arr) == 0:
                    continue
                    # print 'no data is here'
                else:
                    for item, index in rows_arr:
                        if index >= limits[vari_name]['upper'] or 
                                index <= limits[vari_name]['lower']:
                            print 'data exceeds limit: ', index
                        else:
                            continue
                            # print 'data is within range: ', index
        else:
            print 'sorry: this variable name is invalid'

    start = start_day
    dates = [start + dt.timedelta(days=i) for i in range(num)]
    return dates


def main():
    dates = generate_data()


main()