Setting certain elements of a large data frame column to NA is setting all the columns of the data frame to NA

Question

I have a large data frame of almost 186,000 rows with 27 variables. A particular variable "price" has some incorrect values. I need to set incorrect values to NA so that analysis will ignore these values and others will be fine.

I have simplified the problem as below. dput output is also given for reference

dfCSV = structure(list(isin = structure(c(115L, 68L, 100L, 98L, 67L, 
78L, 81L), .Label = c("IN0019780025", "IN0019780074", "IN0019790024", 
"IN0019800021", "IN0019810020", "IN0019820037", "IN0019820128", 
"IN0019830010", "IN0019830143", "IN0019840035", "IN0019840084", 
"IN0019850034", "IN0019850059", "IN0019860033", "IN0019870073", 
"IN0019880031", "IN0019890030", "IN0019900045", "IN0019910036", 
"IN0019910044", "IN0019910127", "IN0019910192", "IN0019950164", 
"IN0019960056", "IN0019960098", "IN0019970014", "IN0019970121", 
"IN0019980021", "IN0019980047", "IN0019980062", "IN0019980096", 
"IN0019980120", "IN0019980153", "IN0019980187", "IN0019980229", 
"IN0019980286", "IN0019980336", "IN0019980344", "IN0019990012", 
"IN0019990038", "IN0019990129", "IN0019990137", "IN0020000025", 
"IN0020000033", "IN0020000041", "IN0020000066", "IN0020000074", 
"IN0020000082", "IN0020000090", "IN0020000108", "IN0020000116", 
"IN0020000124", "IN0020000132", "IN0020010016", "IN0020010024", 
"IN0020010032", "IN0020010040", "IN0020010057", "IN0020010065", 
"IN0020010073", "IN0020010081", "IN0020010099", "IN0020010107", 
"IN0020020015", "IN0020020023", "IN0020020031", "IN0020020049", 
"IN0020020056", "IN0020020064", "IN0020020072", "IN0020020080", 
"IN0020020098", "IN0020020106", "IN0020020122", "IN0020020130", 
"IN0020020155", "IN0020020163", "IN0020020171", "IN0020020213", 
"IN0020020221", "IN0020020247", "IN0020030014", "IN0020030022", 
"IN0020030030", "IN0020030048", "IN0020030055", "IN0020030063", 
"IN0020030097", "IN0020039015", "IN0020039031", "IN0020040013", 
"IN0020040039", "IN0020050012", "IN0020060037", "IN0020060045", 
"IN0020060078", "IN0020060086", "IN0020060219", "IN0020060318", 
"IN0020070010", "IN0020070028", "IN0020070036", "IN0020070044", 
"IN0020070051", "IN0020070069", "IN0020070077", "IN0020080019", 
"IN0020080043", "IN0020080050", "IN0020080068", "IN0020090018", 
"IN0020090026", "IN0020090034", "IN0020090042", "IN0020090059", 
"IN0020090067", "IN0020100015", "IN0020100023", "IN0020100031", 
"IN0020110014", "IN0020110022", "IN0020110030", "IN0020110048", 
"IN0020110055", "IN0020110063", "IN0020110071", "IN0020120013", 
"IN0020120021", "IN0020120039", "IN0020120047", "IN0020120054", 
"IN0020120062"), class = "factor"), tr_date = structure(c(14673, 
13272, 13731, 13515, 13578, 14705, 14155), class = "Date"), ytm = c(40.4806, 
85.041, 80.1207, 75.1705, 82.2098, 70.2422, 10.3402), price = c(25, 
10, 10, 10.19, 9.535, 9.18, 64)), .Names = c("isin", "tr_date", 
"ytm", "price"), row.names = c(11358L, 25878L, 29827L, 43679L, 
75310L, 124470L, 156240L), class = "data.frame")

the following line produces correct output where the first six rows have incorrect prices.

> dfCSV[dfCSV$price < 66, c("isin", "tr_date", "ytm", "price")]
               isin    tr_date     ytm  price
11358  IN0020090059 2010-03-05 40.4806 25.000
25878  IN0020020056 2006-05-04 85.0410 10.000
29827  IN0020070010 2007-08-06 80.1207 10.000
43679  IN0020060219 2007-01-02 75.1705 10.190
75310  IN0020020049 2007-03-06 82.2098  9.535
124470 IN0020020171 2010-04-06 70.2422  9.180
156240 IN0020020247 2008-10-03 10.3402 64.000

Setting the wrong prices to -1, And the output is as expected

> dfCSV$price[dfCSV$price < 50] = -1
> dfCSV[dfCSV$price < 66, c("isin", "tr_date", "ytm", "price")]
               isin    tr_date     ytm price
11358  IN0020090059 2010-03-05 40.4806    -1
25878  IN0020020056 2006-05-04 85.0410    -1
29827  IN0020070010 2007-08-06 80.1207    -1
43679  IN0020060219 2007-01-02 75.1705    -1
75310  IN0020020049 2007-03-06 82.2098    -1
124470 IN0020020171 2010-04-06 70.2422    -1
156240 IN0020020247 2008-10-03 10.3402    64

Now the incorrect price (-1 in this case) is set to NA and the hell breaks loose.

> dfCSV$price[dfCSV$price == -1] = NA
> dfCSV[dfCSV$price < 66, c("isin", "tr_date", "ytm", "price")]
               isin    tr_date     ytm price
NA             <NA>       <NA>      NA    NA
NA.1           <NA>       <NA>      NA    NA
NA.2           <NA>       <NA>      NA    NA
NA.3           <NA>       <NA>      NA    NA
NA.4           <NA>       <NA>      NA    NA
NA.5           <NA>       <NA>      NA    NA
156240 IN0020020247 2008-10-03 10.3402    64

My question is what am I doing wrong here which is causing the rows to disappear in stead of making price points as NA.

I saw a number of posts where usage of lapply is recommended. In my case, I need to change only some of the prices in a given variable. Handling this in R will be a bit tricky and not R-ish. I am sure there is an elegant solution out there. This will be a good way to learn more of R.

Please help me with this issue. regards

Kishore

Answer 1

If you compare with NA the result is NA and not TRUE or FALSE. That messes up your subsetting. You could use dfCSV$price < 66 | is.na(dfCSV$price) (note that NA | TRUE returns TRUE).

dfCSV[dfCSV$price < 66 | is.na(dfCSV$price), c("isin", "tr_date", "ytm", "price")]
#                isin    tr_date     ytm price
# 11358  IN0020090059 2010-03-05 40.4806    NA
# 25878  IN0020020056 2006-05-04 85.0410    NA
# 29827  IN0020070010 2007-08-06 80.1207    NA
# 43679  IN0020060219 2007-01-02 75.1705    NA
# 75310  IN0020020049 2007-03-06 82.2098    NA
# 124470 IN0020020171 2010-04-06 70.2422    NA
# 156240 IN0020020247 2008-10-03 10.3402    64