How to export asynchronously loaded data to be used in a different node module?

Question

I have an issue with loading config variables which are loaded by async call.

the config variables like the following:

//config.js
let config = {
    db: {
        dev: {
            client: ENV_VARS.dev_db_client,
            host: ENV_VARS.dev_db_host,
            name: ENV_VARS.dev_db_name,
            user: ENV_VARS.dev_db_user,
            password: ENV_VARS.dev_db_password,
            min_pool: ENV_VARS.dev_db_min_pool,
            max_pool: ENV_VARS.dev_db_max_pool
        },
        staging: {
            client: ENV_VARS.staging_db_client,
            host: ENV_VARS.staging_db_host,
            name: ENV_VARS.staging_db_name,
            user: ENV_VARS.staging_db_user,
            password: ENV_VARS.staging_db_password,
            min_pool: ENV_VARS.staging_db_min_pool,
            max_pool: ENV_VARS.staging_db_max_pool
        },
        production: async function () {
            let secret = await getSecretFromStore(`SECRET_NAME`);
            let tmpConfig = JSON.parse(secret);
            return {
                    client: ENV_VARS.production_db_client,
                    host: tmpConfig.host,
                    password: tmpConfig.password,
                    user: tmpConfig.username,
                    name: tmpConfig.db_name,
                    min_pool: ENV_VARS.production_db_min_pool,
                    max_pool: ENV_VARS.production_db_max_pool
                };
            });
    },
    NODE_ENV: ENV_VARS.NODE_ENV,
    STACK_ID: ENV_VARS.STACK_ID,
};
module.exports = config;

//knexfile.js
const config = require('authenticator/config');
let productionConfig = {};
if (config.NODE_ENV == 'production') {
// how to load the production vars in the right way and export them?
//the next line is async call and it is wrong to call it like this
    productionConfig = config.db.production();
}
const _config = {
    development: {
        client: config.db.dev.client,
        connection: {
            host: config.db.dev.host,
            database: config.db.dev.name,
            user: config.db.dev.user,
            password: config.db.dev.password,
            // debug: ['ComQueryPacket']
        },
        pool: {
            min: Number(config.db.dev.min_pool) || 0,
            max: Number(config.db.dev.max_pool) || 1
        },
        migrations: {
            tableName: 'knex_migrations'
        }
    },

    staging: {
        client: config.db.staging.client,
        connection: {
            host: config.db.staging.host,
            database: config.db.staging.name,
            user: config.db.staging.user,
            password: config.db.staging.password
        },
        pool: {
            min: Number(config.db.staging.min_pool) || 0,
            max: Number(config.db.staging.max_pool) || 1
        },
        migrations: {
            tableName: 'knex_migrations'
        }
    },

    production: {
        client: productionConfig.client,
        connection: {
            host: productionConfig.host,
            database: productionConfig.name,
            user: productionConfig.user,
            password: productionConfig.password
        },
        pool: {
            min: Number(productionConfig.min_pool) || 0,
            max: Number(productionConfig.max_pool) || 1
        },
        migrations: {
            tableName: 'knex_migrations'
        }
    }
};
module.exports = _config;
//db.js
const config = require('config');
//the config are loaded into knexfile.js and used but production can't be loaded cuz it is async call
//the next line should call the production vars in the env is production
const knexConfig = require('../knexfile')[config.NODE_ENV];
const knex = require('knex')(knexConfig);
module.exports = knex;

So, is there any good strategy to load that module in different node_module?

in another word, it is easy to load staging and dev variables and use them, but since the production variables are loaded by async call, then how to load them in the right way?

If generators are used. will this solve the issue?

Answer 1

The first thing I would do is make the development and staging config have the same semantics as live. Since production returns a promise, dev and staging should return a promise as well. Not doing so is setting yourself up for surprises going to production.

So for instance:

db: {
    dev: async function() {
        await delay(0); // Where `delay` is a `setTimeout` wrapper
        return {
            client: ENV_VARS.dev_db_client,
            host: ENV_VARS.dev_db_host,
            name: ENV_VARS.dev_db_name,
            user: ENV_VARS.dev_db_user,
            password: ENV_VARS.dev_db_password,
            min_pool: ENV_VARS.dev_db_min_pool,
            max_pool: ENV_VARS.dev_db_max_pool
        };
    },
// ....

...and the same for staging. You might even see how long the delay is in production and mimic that in dev and staging. (Unless you're returning all configs as you currently are.)

Your production should also be an async function, since you're using await within it.

Your next step is to de-duplicate that code. You have the same structure being built for all three configs; centralize that in a function:

function buildConfig(env) {
    return {
        client: env.client,
        connection: {
            host: env.host,
            database: env.name,
            user: env.user,
            password: env.password,
            // debug: ['ComQueryPacket']
        },
        pool: {
            min: Number(env.min_pool) || 0,
            max: Number(env.max_pool) || 1
        },
        migrations: {
            tableName: 'knex_migrations'
        }
    };
}

Then, export a promise of the configs, not the actual configs:

module.exports = Promise.all([config.db.dev, config.db.staging, config.db.production])
    .then(([dev, staging, production]) => {
        return {
            development: buildConfig(dev),
            staging:     buildConfig(staging),
            production:  buildConfig(production)
        };
    });

Modules using it will need to consume that promise, since the operation is async (in all environments):

require(/*...*/).then(config => {
    // Code using the config here...
});

---

Before that much longer, this will become less painful (with ESM modules, not CommonJS ones) thanks to the top-level await proposal, which lets module resolution be asynchronous and lets you use await at the top level.