user6020355
user6020355

Reputation:

INSERT operation using Node.js and PostgreSQL doesn't work

I'm creating my first app using Node.js and PostgreSQL. This app connect to db, create table, do web scraping of a web site, insert info to db and then disconnect from db. I'm trying to do using async/await.

The problem is that the insert operation doesn't work. There is no errors, simply table coverages remains empty.

This is my code.

app.js:

const postgreSQLlib = require('./middlewares/postgreSQLlib.js')
const scraper = require('./routers/scraper.js');

const start = async function() {
    await console.log('START');
    
    // connect
    await postgreSQLlib.connect();

    // create tables
    var queryCreateCoverages = {
        text: 'CREATE TABLE IF NOT EXISTS coverages ('+
                'id SERIAL PRIMARY KEY,' +
                'vaccine VARCHAR(64) NOT NULL,' + 
                'country VARCHAR(255) NOT NULL,' +
                'region VARCHAR(255),' +
                'year VARCHAR(4) NOT NULL,' +
                'value VARCHAR(12) NOT NULL);'
    };
    var queryRes = await postgreSQLlib.query(queryCreateCoverages);
    //console.log('Result: ', queryRes);

    // get data
    await scraper.download();

    const query = {
        text: 'SELECT * FROM coverages;',
        values: ['Italy']
    }
    var queryRes = await postgreSQLlib.query(query);
    //console.log('Result: ', queryRes);
    
    // disconnect 
    await postgreSQLlib.disconnect();
     
    return 'FINISH';
}

// start application
start()
    .then(function(res) {
        console.log(res);
    })
    .catch(function(err) {
        console.log(err);
    });

postgreSQLlib.js:

var fs = require('fs');
const {Client} = require('pg'); // node-postgres npm package

const userDb = 'admin';
const passDb = 'admin';
const hostDb = 'localhost'; // default
const portDb = '5432'; // default
const nameDb = 'db'; 
const connectionString = 'postgres://' + userDb + ':' + passDb + '@' + hostDb + ':' + portDb + '/' + nameDb; // result is 'postgres://admin:admin@localhost:5432/db';
let client;

var methods = {};

methods.connect = async function() {
    client = new Client({connectionString});
    return await client.connect()
    .then(async function() {
        await console.log('\nConnected to ' + client.database + ' at ' + client.host + ':' + client.port + ' as ' + client.user + ' (pass: ' + client.password + ')');
    })
    .catch(function(err) {
        console.log('\nError during connection to PostgreSQL');
        throw err;
    });
}

methods.query = function(query) {const start = Date.now();
    return client.query(query)
    .then(function(res) {
        const duration = Date.now() - start;
        console.log('\nExecuted query: {\n  ' + query.text + '\n  [' + query.values + ']' + '\n  duration: '+ duration + '\n  rows: ' + res.rows.length + '\n}');
        return res;
    })
    .catch(function(err) { 
        console.log('\nError executing query', err.stack);
    });
}

methods.disconnect = async function() {
    await client.end()
    .then(function() {
        console.log('\nConnection has ended');
    })
    .catch(function(err) {
        console.log('\nError during clossing connection');
        throw err;
    }); 
}

module.exports = methods;

scraper.js:

var cheerio = require('cheerio');
var request = require('request');
var util = require('../helpers/util.js');
var postgreSQLlib = require('../middlewares/postgreSQLlib.js');

var methods = {};

var countries = {
    'Italy': 'ITA',
    'Latvia': 'LVA',
    'Netherlands': 'NLD'
};

var outDir = './output/';
var finalData = outDir + 'coverage-eu.json'

var jsons = [];

methods.download = async function(req, res) {
    for(country in countries) {
        var url = 'http://apps.who.int/immunization_monitoring/globalsummary/coverages?c=' + countries[country];

        return await request(url, (async function(country) {
            var thisCountry = country;

            return async function(error, res, html) {
                if(error) {
                    throw error;
                }

                $ = cheerio.load(html);

                var years = [];
                var vaccines = [];
                var coverages = [];

                $('.ts .year').each(function() {
                    years.push($(this).text().trim());
                });
                $('.ts .odd td a, .ts .even td a').each(function() {
                    vaccines.push($(this).text().trim());
                });
                $('.ts .odd .statistics_small, .ts .even .statistics_small').each(function() {
                    coverages.push($(this).text().trim());
                });

                const numYears = years.length;
                const numVaccines = vaccines.length;
                for(var vaccineIdx = 0; vaccineIdx < numVaccines; vaccineIdx++) {
                    for(var yearIdx = 0; yearIdx < numYears; yearIdx++) {
                        // save on db
                        const query = {
                            text: 'INSERT INTO coverages (vaccine, country, region, year, value) VALUES ($1, $2, $3, $4, $5);',
                            values: [vaccines[vaccineIdx], country, '', years[yearIdx], coverages[vaccineIdx*numYears + yearIdx]]
                        }
                        var queryRes = await postgreSQLlib.query(query);
                        //console.log('Result: ', queryRes);
                    } 
                }
            } 
        })(country));
    } 
};

module.exports = methods;

Why? I can't solve the error if no errors are showed.

Any help is appreciate and sorry for my bad english.


EDIT 1

If I use SQL shell (psql) to insert records, it works:

enter image description here


EDIT 2

I modified the console.log of the query method:

methods.query = async function(query, print) {
    return await client.query(query)
    .then(function(res) {
        console.log(print, 'OK query');
        return res;
    })
    .catch(function(err) { 
        console.log(print, 'ERR query');
    });
}

scraper.js:

methods.download = async function(req, res) {
    for(country in countries) {
        console.log('\nCOUNTRY:', country); // ADDED
        var url = 'http://apps.who.int/immunization_monitoring/globalsummary/coverages?c=' + countries[country];
        let res = await request(url);
        insert(res);
    }
}

//...

// save on db
const queryInsert = {
    text: 'INSERT INTO coverages (vaccine, country, region, year, value) VALUES($1, $2, $3, $4, $5);',
    values: [vaccines[vaccineIdx], country, '', years[yearIdx], coverages[vaccineIdx*numYears + yearIdx]]
}
var printText = '[INSERT ' + country + ' IN coverages]';
var queryRes = await postgreSQLlib.query(queryInsert, printText);

These are the prints:

START

Connected to db at localhost:5432 as admin (pass: admin)
[CREATE TABLE coverages] OK query

COUNTRY: Italy

COUNTRY: Latvia
[INSERT Italy IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query

COUNTRY: Netherlands
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query
[INSERT Netherlands IN coverages] OK query

Connection has ended

FINISH
[INSERT Netherlands IN coverages] ERR query

As you can see, there is an error at the end because the connection is already closed. In addition, prints related to the query result are not ordered with respect to the country. For example:

COUNTRY: Italy

COUNTRY: Latvia
[INSERT Italy IN coverages] OK query
[INSERT Latvia IN coverages] OK query
[INSERT Latvia IN coverages] OK query

There is nothing below COUNTRY: Italy, but results about Italy queries are below COUNTRY: Latvia (and there are not all).

Upvotes: 9

Views: 3915

Answers (1)

Mustafa Mamun
Mustafa Mamun

Reputation: 2661

There are two mistakes in app.js file.

const postgreSQLlib = require('./middlewares/postgreSQLlib.js')
const downloaderCoverageIta = require('./routers/scraper.js');

const start = async function() {
    // connect
    await postgreSQLlib.connect();

    // create tables
    var queryCreateCoverages = {
        text: 'CREATE TABLE IF NOT EXISTS coverages ('+
                    'vaccine VARCHAR(64),' + 
                    'country VARCHAR(255),' +
                    'region VARCHAR(255),' +
                    'year VARCHAR(4),' +
                    'value VARCHAR(12),' +
                    'PRIMARY KEY(vaccine, country, region, year))'
    };
    var queryRes = await postgreSQLlib.query(queryCreateCoverages);
    //console.log('Result: ', queryRes);

    // get data
    await downloaderCoverageIta.download();

    const query = {
        text: 'SELECT * FROM coverages WHERE country = $1',
        values: ['Italy']
    }
    return await postgreSQLlib.query(query);
    //console.log('Result: ', queryRes);

    // disconnect 
    await postgreSQLlib.disconnect();
}

// start application
start()
.then(function(res) {
    console.log(res);
})
.catch(function(err) {
    console.log(err);
});

First you were not using right var "downloaderCoverageIta" then you were not returning the value.

And few in srcapper.js file

var cheerio = require('cheerio');
var request = require('request-promise');
var util = require('../helpers/util.js');
var postgreSQLlib = require('./postgreSQLlib.js');

var methods = {};

var countries = {
'Italy': 'ITA',
'Latvia': 'LVA',
'Netherlands': 'NLD'
};

var outDir = './output/';
var finalData = outDir + 'coverage-eu.json'

var jsons = [];

methods.download = async function(req, res) {
await Promise.all(Object.values(countries).map(async country=> {
    var url = 
'http://apps.who.int/immunization_monitoring/globalsummary/coverages?c=' + country;

    const html = await request(url);
    $ = cheerio.load(html);

    var years = [];
    var vaccines = [];
    var coverages = [];

    $('.ts .year').each(function() {
        years.push($(this).text().trim());
    });
    $('.ts .odd td a, .ts .even td a').each(function() {
        vaccines.push($(this).text().trim());
    });
    $('.ts .odd .statistics_small, .ts .even 
.statistics_small').each(function() {
        coverages.push($(this).text().trim());
    });

    const numYears = years.length;
    const numVaccines = vaccines.length;
    for(var vaccineIdx = 0; vaccineIdx < numVaccines; vaccineIdx++) {
        for(var yearIdx = 0; yearIdx < numYears; yearIdx++) {
            let obj = {
                year: years[yearIdx],
                country: country,
                region: "",
                vaccine: vaccines[vaccineIdx],
                coverage: coverages[vaccineIdx*numYears + yearIdx]
            }

            // save on db
            const query = {
                text: 'INSERT INTO coverages (vaccine, country, 
region, year, value) VALUES($1, $2, $3, $4, $5)',
                values: [vaccines[vaccineIdx], country, '', 
years[yearIdx], coverages[vaccineIdx*numYears + yearIdx]],
            }
            var queryRes = await postgreSQLlib.query(query);

            // save on file
            jsons.push(obj);
        }
    }
    await util.printOnFile(jsons, finalData);
}))
}

module.exports = methods;

As per I know request module does not work with the async await. This worked for me atleast.

enter image description here This is from my pg admin after i run the code.

Upvotes: 1

Related Questions