"Can't convert float Nan to int" but no Nan?

Question

I have a dataframe and attempt to make the following operation:

data['SD_rates']=np.array([int((data['actual value'][i]-data['means'][i])/data['std'][i]) for i in range (len(data['means']))])  

It breaks with the following message: "Can't convert float Nan to int"

It is an error I understand but tested the df with data.isnull() and no column involved includes NaN (I controlled it manually by sending data.to_csv).

I even filled data['std'] with fillna(-1, inplace=True) but still, it breaks. I don't understand why, since there is no division by 0 (i also controlled that there were no zeros in this column, so no original 0 and Null/Nan filled with -1), and actual values and means are fillna(0) for missing values, and anyway the substraction can't produce a nan (data range in [0-10]).

What could be wrong? (as i said, the data right before triggering the operation is correct...). Thanks

---

Here is a code snippet:
One of my hypotheses is that in some way, groupby might generate NaN, that I can't get rid off when calculating my means (but I believed that it was ignored by pandas automatically...) and that are not filled with 0 or -1 (I chose -1 for standard deviation deliberately to avoid dividing by 0).

def stats_setting(data):

    print('Stats settings')
    print(data.columns)
    print(data.dtypes) 
    #sys.exit()

    data['marks']=np.log1p(data['marks'].astype(float))
    data['students']=np.log1p(data['students'].astype(float))#Rossman9 think this has to be tested
    #were filled with fillna before)

#First Part: by studentType and Assortment
    types_DoM_select=['Type','Type2','Category']

#First Block:types_DoM students grouped by categories
#wonder if can do a groupby of groupb
    print("types_DoM_marks_means")
    types_DoM_marks_means = data.groupby(types_DoM_select)['marks'].mean()
    types_DoM_marks_means.name = 'types_DoM_marks_means'
    types_DoM_marks_means = types_DoM_marks_means.reset_index()
    data = pd.merge(data, types_DoM_marks_means, on = types_DoM_select, how='left')

    print("types_DoM_students_means")
    types_DoM_students_means = data.groupby(types_DoM_select)['students'].mean() #.students won't work. Why?
    types_DoM_students_means.name = 'types_DoM_students_means'
    types_DoM_students_means=types_DoM_students_means.reset_index()    
    data = pd.merge(data, types_DoM_students_means, on = types_DoM_select, how='left')

    print("types_DoM_marks_medians")
    types_DoM_marks_medians = data.groupby(types_DoM_select)['marks'].median()
    types_DoM_marks_medians.name = 'types_DoM_marks_medians'
    types_DoM_marks_medians = types_DoM_marks_medians.reset_index()
    data = pd.merge(data, types_DoM_marks_medians, on = types_DoM_select, how='left')

    print("types_DoM_students_medians")
    types_DoM_students_medians = data.groupby(types_DoM_select)['students'].median() #.students won't work. Why?
    types_DoM_students_medians.name = 'types_DoM_students_medians'
    types_DoM_students_medians=types_DoM_students_medians.reset_index()    
    data = pd.merge(data, types_DoM_students_medians, on = types_DoM_select, how='left')
    print("types_DoM_marks_std")
    types_DoM_marks_std = data.groupby(types_DoM_select)['marks'].std()
    types_DoM_marks_std.name = 'types_DoM_marks_std'
    types_DoM_marks_std = types_DoM_marks_std.reset_index()
    data = pd.merge(data, types_DoM_marks_std, on = types_DoM_select, how='left')


    print("types_DoM_students_std")
    types_DoM_students_std = data.groupby(types_DoM_select)['students'].std()
    types_DoM_students_std.name = 'types_DoM_students_std'
    types_DoM_students_std = types_DoM_students_std.reset_index()
    data = pd.merge(data, types_DoM_students_std, on = types_DoM_select, how='left')

    data['types_DoM_marks_means'].fillna(-1, inplace=True)
    data['types_DoM_students_means'].fillna(-1, inplace=True)
    data['types_DoM_marks_medians'].fillna(-1, inplace=True)
    data['types_DoM_students_medians'].fillna(-1, inplace=True)
    data['types_DoM_marks_std'].fillna(-1, inplace=True)
    data['types_DoM_students_std'].fillna(-1, inplace=True)

#Second Part: by specific student
    student_DoM_select=['Type','Type2','Category']

#First Block:student_DoM
#wonder if can do a groupby of groupb
    print("student_DoM_marks_means")
    student_DoM_marks_means = data.groupby(student_DoM_select)['marks'].mean()
    student_DoM_marks_means.name = 'student_DoM_marks_means'
    student_DoM_marks_means = student_DoM_marks_means.reset_index()
    data = pd.merge(data, student_DoM_marks_means, on = student_DoM_select, how='left')

    print("student_DoM_students_means")
    student_DoM_students_means = data.groupby(student_DoM_select)['students'].mean() #.students won't work. Why?
    student_DoM_students_means.name = 'student_DoM_students_means'
    student_DoM_students_means=student_DoM_students_means.reset_index()    
    data = pd.merge(data, student_DoM_students_means, on = student_DoM_select, how='left')

    print("student_DoM_marks_medians")
    student_DoM_marks_medians = data.groupby(student_DoM_select)['marks'].median()
    student_DoM_marks_medians.name = 'student_DoM_marks_medians'
    student_DoM_marks_medians = student_DoM_marks_medians.reset_index()
    data = pd.merge(data, student_DoM_marks_medians, on = student_DoM_select, how='left')

    print("student_DoM_students_medians")
    student_DoM_students_medians = data.groupby(student_DoM_select)['students'].median() #.students won't work. Why?
    student_DoM_students_medians.name = 'student_DoM_students_medians'
    student_DoM_students_medians=student_DoM_students_medians.reset_index()    
    data = pd.merge(data, student_DoM_students_medians, on = student_DoM_select, how='left')

    # May I use data['marks','students','marksMean','studentsMean','marksMedian','studentsMedian']=data['marks','students','marksMean','studentsMean','marksMedian','studentsMedian'].astype(int) to spare memory?

    print("student_DoM_marks_std")
    student_DoM_marks_std = data.groupby(student_DoM_select)['marks'].std()
    student_DoM_marks_std.name = 'student_DoM_marks_std'
    student_DoM_marks_std = student_DoM_marks_std.reset_index()
    data = pd.merge(data, student_DoM_marks_std, on = student_DoM_select, how='left')

    print("student_DoM_students_std")
    student_DoM_students_std = data.groupby(student_DoM_select)['students'].std()
    student_DoM_students_std.name = 'student_DoM_students_std'
    student_DoM_students_std = student_DoM_students_std.reset_index()
    data = pd.merge(data, student_DoM_students_std, on = student_DoM_select, how='left')

    data['student_DoM_marks_means'].fillna(0, inplace=True)
    data['student_DoM_students_means'].fillna(0, inplace=True)
    data['student_DoM_marks_medians'].fillna(0, inplace=True)
    data['student_DoM_students_medians'].fillna(0, inplace=True)
    data['student_DoM_marks_std'].fillna(0, inplace=True)
    data['student_DoM_students_std'].fillna(0, inplace=True)

#Third Part: Exceptional students 

    #I think int is better here as it helps defining categories but can't use it.#    
    #print(data.isnull().sum())
    #print(data['types_DoM_marks_std'][data['types_DoM_marks_std']==0].sum())
    #data.to_csv('ex')
    #print(data.columns)

#Original version:#int raises the "can't convert Nan float to int. While there were no Nan as I verified in the data just before sending it to the    
    data['Except_student_IP2_DoM_marks_means']=np.array([int((data['student_IP2_DoM_marks_means'][i]-data['types_IP2_DoM_marks_means'][i])/data['types_IP2_DoM_students_std'][i]) for i in range (len(data['year']))])  
    data['Except_student_IP2_DoM_marks_medians']=np.array([int((data['student_IP2_DoM_marks_medians'][i]-data['types_IP2_DoM_marks_means'][i])/data['types_IP2_DoM_students_std'][i]) for i in range (len(data['year']))])  
#Second version: raises no error but final data (returned) is filled with these stupid NaN
    data['Except_student_P2M_DoM_marks_means']=np.array([np.round((data['student_DoM_marks_means'][i]-data['types_DoM_marks_means'][i])/data['types_DoM_marks_std'][i],0) for i in range (len(data['year']))])  
    data['Except_student_P2M_DoM_marks_medians']=np.array([np.round((data['student_DoM_marks_medians'][i]-data['types_DoM_marks_medians'][i])/data['types_DoM_marks_std'][i],0) for i in range (len(data['year']))])  

#End
    return data

Answer 1

Most likely you are correct that there are no Nans in your data frame, however you are creating them in your calculations. See the following:

In [15]: import pandas as pd
In [16]: df = pd.DataFrame([[1, 2], [0, 0]], columns=['actual value', 'col2'])
         df['means'] = df.mean(axis=1)
         df['std'] = df.std(axis=1)

In [17]: df
Out[17]:
   actual value  col2  means  std
0             1     2    1.5  0.5
1             0     0    0.0  0.0

So the data frame doesn't have any Nans, but what about the calculations?

In [21]: [(df['actual value'][i]-df['means'][i])/df['std'][i] for i in range (len(df['means']))]
Out[21]: [-1.0, nan]

Now when you call int on that you get an error on the resulting list. Finally, I would suggest (if possible) performing the operations directly in the underlying arrays rather then using a for loop, as it will be much faster.

In [25]: (df['actual value']-df['means'])/df['std']
Out[25]:
0    -1
1   NaN
dtype: float64

This may not be possible depending on what return value of a 0 division is desired though.