Nathalie
Reputation: 1238
Reading files from a file
From this project
I try to run this code
But it has a problem when it reads the files python process_data.py ./GoogleNews-vectors-negative300.bin ./essays.csv ./mairesse.csv
However I receive this error:
C:\personality-detection-master>python process_data.py ./GoogleNews-vectors-nega
tive300.bin ./essays.csv ./mairesse.csv
WARNING (theano.configdefaults): g++ not available, if using conda: `conda insta
ll m2w64-toolchain`
C:\Users\nathalie\Miniconda3\lib\site-packages\theano\configdefaults.py:560: UserWa
rning: DeprecationWarning: there is no c++ compiler.This is deprecated and with
Theano 0.11 a c++ compiler will be mandatory
warnings.warn("DeprecationWarning: there is no c++ compiler."
WARNING (theano.configdefaults): g++ not detected ! Theano will be unable to exe
cute optimized C-implementations (for both CPU and GPU) and will default to Pyth
on implementations. Performance will be severely degraded. To remove this warnin
g, set Theano flags cxx to an empty string.
WARNING (theano.tensor.blas): Using NumPy C-API based implementation for BLAS fu
nctions.
Traceback (most recent call last):
File "process_data.py", line 163, in <module>
revs, vocab = build_data_cv(data_folder, cv=10, clean_string=True)
File "process_data.py", line 21, in build_data_cv
for line in csvreader:
_csv.Error: iterator should return strings, not bytes (did you open the file in
text mode?)
For this code:
import numpy as np
import theano
import _pickle as cPickle
from collections import defaultdict
import sys, re
import pandas as pd
import csv
import getpass
def build_data_cv(datafile, cv=10, clean_string=True):
"""
Loads data and split into 10 folds.
"""
revs = []
vocab = defaultdict(float)
with open(datafile, "rb") as csvf:
csvreader=csv.reader(csvf,delimiter=',',quotechar='"')
first_line=True
for line in csvreader:
if first_line:
first_line=False
continue
status=[]
sentences=re.split(r'[.?]', line[1].strip())
try:
sentences.remove('')
except ValueError:
None
for sent in sentences:
if clean_string:
orig_rev = clean_str(sent.strip())
if orig_rev=='':
continue
words = set(orig_rev.split())
splitted = orig_rev.split()
if len(splitted)>150:
orig_rev=[]
splits=int(np.floor(len(splitted)/20))
for index in range(splits):
orig_rev.append(' '.join(splitted[index*20:(index+1)*20]))
if len(splitted)>splits*20:
orig_rev.append(' '.join(splitted[splits*20:]))
status.extend(orig_rev)
else:
status.append(orig_rev)
else:
orig_rev = sent.strip().lower()
words = set(orig_rev.split())
status.append(orig_rev)
for word in words:
vocab[word] += 1
datum = {"y0":1 if line[2].lower()=='y' else 0,
"y1":1 if line[3].lower()=='y' else 0,
"y2":1 if line[4].lower()=='y' else 0,
"y3":1 if line[5].lower()=='y' else 0,
"y4":1 if line[6].lower()=='y' else 0,
"text": status,
"user": line[0],
"num_words": np.max([len(sent.split()) for sent in status]),
"split": np.random.randint(0,cv)}
revs.append(datum)
return revs, vocab
def get_W(word_vecs, k=300):
"""
Get word matrix. W[i] is the vector for word indexed by i
"""
vocab_size = len(word_vecs)
word_idx_map = dict()
W = np.zeros(shape=(vocab_size+1, k), dtype=theano.config.floatX)
W[0] = np.zeros(k, dtype=theano.config.floatX)
i = 1
for word in word_vecs:
W[i] = word_vecs[word]
word_idx_map[word] = i
i += 1
return W, word_idx_map
def load_bin_vec(fname, vocab):
"""
Loads 300x1 word vecs from Google (Mikolov) word2vec
"""
word_vecs = {}
with open(fname, "rb") as f:
header = f.readline()
vocab_size, layer1_size = map(int, header.split())
binary_len = np.dtype(theano.config.floatX).itemsize * layer1_size
for line in xrange(vocab_size):
word = []
while True:
ch = f.read(1)
if ch == ' ':
word = ''.join(word)
break
if ch != '\n':
word.append(ch)
if word in vocab:
word_vecs[word] = np.fromstring(f.read(binary_len), dtype=theano.config.floatX)
else:
f.read(binary_len)
return word_vecs
def add_unknown_words(word_vecs, vocab, min_df=1, k=300):
"""
For words that occur in at least min_df documents, create a separate word vector.
0.25 is chosen so the unknown vectors have (approximately) same variance as pre-trained ones
"""
for word in vocab:
if word not in word_vecs and vocab[word] >= min_df:
word_vecs[word] = np.random.uniform(-0.25,0.25,k)
#print word
def clean_str(string, TREC=False):
"""
Tokenization/string cleaning for all datasets except for SST.
Every dataset is lower cased except for TREC
"""
string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
string = re.sub(r"\'s", " \'s ", string)
string = re.sub(r"\'ve", " have ", string)
string = re.sub(r"n\'t", " not ", string)
string = re.sub(r"\'re", " are ", string)
string = re.sub(r"\'d" , " would ", string)
string = re.sub(r"\'ll", " will ", string)
string = re.sub(r",", " , ", string)
string = re.sub(r"!", " ! ", string)
string = re.sub(r"\(", " ( ", string)
string = re.sub(r"\)", " ) ", string)
string = re.sub(r"\?", " \? ", string)
# string = re.sub(r"[a-zA-Z]{4,}", "", string)
string = re.sub(r"\s{2,}", " ", string)
return string.strip() if TREC else string.strip().lower()
def clean_str_sst(string):
"""
Tokenization/string cleaning for the SST dataset
"""
string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
string = re.sub(r"\s{2,}", " ", string)
return string.strip().lower()
def get_mairesse_features(file_name):
feats={}
with open(file_name, "rb") as csvf:
csvreader=csv.reader(csvf,delimiter=',',quotechar='"')
for line in csvreader:
feats[line[0]]=[float(f) for f in line[1:]]
return feats
if __name__=="__main__":
w2v_file = sys.argv[1]
data_folder = sys.argv[2]
mairesse_file = sys.argv[3]
#print "loading data...",
revs, vocab = build_data_cv(data_folder, cv=10, clean_string=True)
num_words=pd.DataFrame(revs)["num_words"]
max_l = np.max(num_words)
print ("data loaded!")
print ("number of status: " + str(len(revs)))
print ("vocab size: " + str(len(vocab)))
print ("max sentence length: " + str(max_l))
print ("loading word2vec vectors...")
w2v = load_bin_vec(w2v_file, vocab)
print ("word2vec loaded!")
print ("num words already in word2vec: " + str(len(w2v)))
add_unknown_words(w2v, vocab)
W, word_idx_map = get_W(w2v)
rand_vecs = {}
add_unknown_words(rand_vecs, vocab)
W2, _ = get_W(rand_vecs)
mairesse = get_mairesse_features(mairesse_file)
cPickle.dump([revs, W, W2, word_idx_map, vocab, mairesse], open("essays_mairesse.p", "wb"))
print ("dataset created!")
I don't have open anywhere the file. What can i do?
Should I make any update to the code?
After changing in the previous code the rb to r
C:\personality-detection-master>python process_data.py ./GoogleNews-vectors-nega
tive300.bin ./essays.csv ./mairesse.csv
WARNING (theano.configdefaults): g++ not available, if using conda: `conda insta
ll m2w64-toolchain`
C:\Users\nathalie\Miniconda3\lib\site-packages\theano\configdefaults.py:560: UserWa
rning: DeprecationWarning: there is no c++ compiler.This is deprecated and with
Theano 0.11 a c++ compiler will be mandatory
warnings.warn("DeprecationWarning: there is no c++ compiler."
WARNING (theano.configdefaults): g++ not detected ! Theano will be unable to exe
cute optimized C-implementations (for both CPU and GPU) and will default to Pyth
on implementations. Performance will be severely degraded. To remove this warnin
g, set Theano flags cxx to an empty string.
WARNING (theano.tensor.blas): Using NumPy C-API based implementation for BLAS fu
nctions.
Traceback (most recent call last):
File "process_data.py", line 163, in <module>
revs, vocab = build_data_cv(data_folder, cv=10, clean_string=True)
File "process_data.py", line 21, in build_data_cv
for line in csvreader:
File "C:\Users\nathalie\Miniconda3\lib\encodings\cp1253.py", line 23, in decode
return codecs.charmap_decode(input,self.errors,decoding_table)[0]
UnicodeDecodeError: 'charmap' codec can't decode byte 0x8e in position 1858: cha
racter maps to <undefined>
Upvotes: 1
Views: 151
Answers (1)
Simon
Reputation: 5698
The error is:
_csv.Error: iterator should return strings, not bytes (did you open the file in
text mode?)
You opened the file in binary mode ("rb"), and you should open it in text mode ("r").
Upvotes: 1