Reputation:
I'm creating my first app using Node.js and PostgreSQL.
This app connect to db, create table, do web scraping of a web site, insert info to db and then disconnect from db.
I'm trying to do using async/await
.
The problem is that the insert operation doesn't work. There is no errors, simply table coverages remains empty.
This is my code.
app.js:
const postgreSQLlib = require('./middlewares/postgreSQLlib.js')
const scraper = require('./routers/scraper.js');
const start = async function() {
await console.log('START');
// connect
await postgreSQLlib.connect();
// create tables
var queryCreateCoverages = {
text: 'CREATE TABLE IF NOT EXISTS coverages ('+
'id SERIAL PRIMARY KEY,' +
'vaccine VARCHAR(64) NOT NULL,' +
'country VARCHAR(255) NOT NULL,' +
'region VARCHAR(255),' +
'year VARCHAR(4) NOT NULL,' +
'value VARCHAR(12) NOT NULL);'
};
var queryRes = await postgreSQLlib.query(queryCreateCoverages);
//console.log('Result: ', queryRes);
// get data
await scraper.download();
const query = {
text: 'SELECT * FROM coverages;',
values: ['Italy']
}
var queryRes = await postgreSQLlib.query(query);
//console.log('Result: ', queryRes);
// disconnect
await postgreSQLlib.disconnect();
return 'FINISH';
}
// start application
start()
.then(function(res) {
console.log(res);
})
.catch(function(err) {
console.log(err);
});
postgreSQLlib.js:
var fs = require('fs');
const {Client} = require('pg'); // node-postgres npm package
const userDb = 'admin';
const passDb = 'admin';
const hostDb = 'localhost'; // default
const portDb = '5432'; // default
const nameDb = 'db';
const connectionString = 'postgres://' + userDb + ':' + passDb + '@' + hostDb + ':' + portDb + '/' + nameDb; // result is 'postgres://admin:admin@localhost:5432/db';
let client;
var methods = {};
methods.connect = async function() {
client = new Client({connectionString});
return await client.connect()
.then(async function() {
await console.log('\nConnected to ' + client.database + ' at ' + client.host + ':' + client.port + ' as ' + client.user + ' (pass: ' + client.password + ')');
})
.catch(function(err) {
console.log('\nError during connection to PostgreSQL');
throw err;
});
}
methods.query = function(query) {const start = Date.now();
return client.query(query)
.then(function(res) {
const duration = Date.now() - start;
console.log('\nExecuted query: {\n ' + query.text + '\n [' + query.values + ']' + '\n duration: '+ duration + '\n rows: ' + res.rows.length + '\n}');
return res;
})
.catch(function(err) {
console.log('\nError executing query', err.stack);
});
}
methods.disconnect = async function() {
await client.end()
.then(function() {
console.log('\nConnection has ended');
})
.catch(function(err) {
console.log('\nError during clossing connection');
throw err;
});
}
module.exports = methods;
scraper.js:
var cheerio = require('cheerio');
var request = require('request');
var util = require('../helpers/util.js');
var postgreSQLlib = require('../middlewares/postgreSQLlib.js');
var methods = {};
var countries = {
'Italy': 'ITA',
'Latvia': 'LVA',
'Netherlands': 'NLD'
};
var outDir = './output/';
var finalData = outDir + 'coverage-eu.json'
var jsons = [];
methods.download = async function(req, res) {
for(country in countries) {
var url = 'http://apps.who.int/immunization_monitoring/globalsummary/coverages?c=' + countries[country];
return await request(url, (async function(country) {
var thisCountry = country;
return async function(error, res, html) {
if(error) {
throw error;
}
$ = cheerio.load(html);
var years = [];
var vaccines = [];
var coverages = [];
$('.ts .year').each(function() {
years.push($(this).text().trim());
});
$('.ts .odd td a, .ts .even td a').each(function() {
vaccines.push($(this).text().trim());
});
$('.ts .odd .statistics_small, .ts .even .statistics_small').each(function() {
coverages.push($(this).text().trim());
});
const numYears = years.length;
const numVaccines = vaccines.length;
for(var vaccineIdx = 0; vaccineIdx < numVaccines; vaccineIdx++) {
for(var yearIdx = 0; yearIdx < numYears; yearIdx++) {
// save on db
const query = {
text: 'INSERT INTO coverages (vaccine, country, region, year, value) VALUES ($1, $2, $3, $4, $5);',
values: [vaccines[vaccineIdx], country, '', years[yearIdx], coverages[vaccineIdx*numYears + yearIdx]]
}
var queryRes = await postgreSQLlib.query(query);
//console.log('Result: ', queryRes);
}
}
}
})(country));
}
};
module.exports = methods;
Why? I can't solve the error if no errors are showed.
Any help is appreciate and sorry for my bad english.
If I use SQL shell (psql) to insert records, it works:
I modified the console.log
of the query
method:
methods.query = async function(query, print) {
return await client.query(query)
.then(function(res) {
console.log(print, 'OK query');
return res;
})
.catch(function(err) {
console.log(print, 'ERR query');
});
}
scraper.js:
methods.download = async function(req, res) {
for(country in countries) {
console.log('\nCOUNTRY:', country); // ADDED
var url = 'http://apps.who.int/immunization_monitoring/globalsummary/coverages?c=' + countries[country];
let res = await request(url);
insert(res);
}
}
//...
// save on db
const queryInsert = {
text: 'INSERT INTO coverages (vaccine, country, region, year, value) VALUES($1, $2, $3, $4, $5);',
values: [vaccines[vaccineIdx], country, '', years[yearIdx], coverages[vaccineIdx*numYears + yearIdx]]
}
var printText = '[INSERT ' + country + ' IN coverages]';
var queryRes = await postgreSQLlib.query(queryInsert, printText);
These are the prints:
START
Connected to db at localhost:5432 as admin (pass: admin)
[CREATE TABLE coverages] OK query
COUNTRY: Italy
COUNTRY: Latvia
[INSERT Italy IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
COUNTRY: Netherlands
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
Connection has ended
FINISH
[INSERT Netherlands IN coverages] ERR query
As you can see, there is an error at the end because the connection is already closed. In addition, prints related to the query result are not ordered with respect to the country. For example:
COUNTRY: Italy
COUNTRY: Latvia
[INSERT Italy IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
There is nothing below COUNTRY: Italy, but results about Italy queries are below COUNTRY: Latvia (and there are not all).
Upvotes: 9
Views: 3915
Reputation: 2661
There are two mistakes in app.js file.
const postgreSQLlib = require('./middlewares/postgreSQLlib.js')
const downloaderCoverageIta = require('./routers/scraper.js');
const start = async function() {
// connect
await postgreSQLlib.connect();
// create tables
var queryCreateCoverages = {
text: 'CREATE TABLE IF NOT EXISTS coverages ('+
'vaccine VARCHAR(64),' +
'country VARCHAR(255),' +
'region VARCHAR(255),' +
'year VARCHAR(4),' +
'value VARCHAR(12),' +
'PRIMARY KEY(vaccine, country, region, year))'
};
var queryRes = await postgreSQLlib.query(queryCreateCoverages);
//console.log('Result: ', queryRes);
// get data
await downloaderCoverageIta.download();
const query = {
text: 'SELECT * FROM coverages WHERE country = $1',
values: ['Italy']
}
return await postgreSQLlib.query(query);
//console.log('Result: ', queryRes);
// disconnect
await postgreSQLlib.disconnect();
}
// start application
start()
.then(function(res) {
console.log(res);
})
.catch(function(err) {
console.log(err);
});
First you were not using right var "downloaderCoverageIta" then you were not returning the value.
And few in srcapper.js file
var cheerio = require('cheerio');
var request = require('request-promise');
var util = require('../helpers/util.js');
var postgreSQLlib = require('./postgreSQLlib.js');
var methods = {};
var countries = {
'Italy': 'ITA',
'Latvia': 'LVA',
'Netherlands': 'NLD'
};
var outDir = './output/';
var finalData = outDir + 'coverage-eu.json'
var jsons = [];
methods.download = async function(req, res) {
await Promise.all(Object.values(countries).map(async country=> {
var url =
'http://apps.who.int/immunization_monitoring/globalsummary/coverages?c=' + country;
const html = await request(url);
$ = cheerio.load(html);
var years = [];
var vaccines = [];
var coverages = [];
$('.ts .year').each(function() {
years.push($(this).text().trim());
});
$('.ts .odd td a, .ts .even td a').each(function() {
vaccines.push($(this).text().trim());
});
$('.ts .odd .statistics_small, .ts .even
.statistics_small').each(function() {
coverages.push($(this).text().trim());
});
const numYears = years.length;
const numVaccines = vaccines.length;
for(var vaccineIdx = 0; vaccineIdx < numVaccines; vaccineIdx++) {
for(var yearIdx = 0; yearIdx < numYears; yearIdx++) {
let obj = {
year: years[yearIdx],
country: country,
region: "",
vaccine: vaccines[vaccineIdx],
coverage: coverages[vaccineIdx*numYears + yearIdx]
}
// save on db
const query = {
text: 'INSERT INTO coverages (vaccine, country,
region, year, value) VALUES($1, $2, $3, $4, $5)',
values: [vaccines[vaccineIdx], country, '',
years[yearIdx], coverages[vaccineIdx*numYears + yearIdx]],
}
var queryRes = await postgreSQLlib.query(query);
// save on file
jsons.push(obj);
}
}
await util.printOnFile(jsons, finalData);
}))
}
module.exports = methods;
As per I know request module does not work with the async await. This worked for me atleast.
This is from my pg admin after i run the code.
Upvotes: 1