Reputation: 3
I am writing a scraper in nodejs with Axios which is intended to access a list of urls and then save it to file with fs. However fs writes [object Promise] to the file instead of the intended html response. Here is the code:
var urls = [url1, url2, url3]
var responses = [];
var completed_requests = 0;
for (i in urls) {
responses.push(axios.get(urls[i]))
completed_requests ++
if(completed_requests == urls.length){
Promise.all(responses).then((e)=>{
for (i in e) {
console.log(responses[i]) //this returns the http correctly
return fs.writeFile('/answer/'+ i +'.txt', responses[i], function(err) {
if(err) {
return console.log(err);
}})
}
}
)}
}
Upvotes: 0
Views: 1735
Reputation: 695
It is writing '[object promise]' because you are writing promise object to the file and also don't return value from for loop if you do so it will run only once . If you want write only html then following code will work.
var urls = [url1,url2,url3]
var responses = [];
var completed_requests = 0;
for (i in urls) {
responses.push(axios.get(urls[i]))
completed_requests ++
if(completed_requests == urls.length){
console.log('here');
Promise.all(responses).then((e)=>{
for (i in e) {
fs.writeFile( i +'.txt', e[i].data, function(err) {
if(err) {
console.log(err);
}})
}
}
)}
}
If you want to write whole object use JSON.stringify(e[i])
.
Upvotes: 1
Reputation: 1698
I think you have first to fill the array so you code is working unexpected just because of a curly brace and because your are saving the Promise object instead the response payload. Please note the comments:
var urls = [url1, url2, url3]
var responses = [];
//var completed requests not needed as a flag here
for (i in urls) {
responses.push(axios.get(urls[i]))
}
//then after having an array of Promises you should then wait for them
//also you should be saving the `e` variable instead of the responses (array) that is a promise.
Promise.all(responses).then((e)=>{
for (i in e) {
console.log(responses[i]) //this returns the http correctly
return fs.writeFile('/answer/'+ i +'.txt', e, function(err) {
if(err) {
return console.log(err);
}})
}
}
)}
Upvotes: 0