CODEWITHSUNDEEP
amazon-web-servicesaws-lambdaterraformterraform-provider-aws
WorkoutBuddy
WorkoutBuddy

Reputation: 759

Terraform lambda invocation probable timeout

After having spent 7 hours on this, I decided to reach out to you. I need to update credentials within in a terraform flow. Since the secrets shall not be in the state-file, I use an AWS lambda function to update the secret of the RDS instance. The password is passed via CLI.

   locals {
  db_password = tostring(var.db_user_password)
}

data "aws_lambda_invocation" "credentials_manager" {
  function_name = "credentials-manager"
  input = <<JSON
{
  "secretString": "{\"username\":\"${module.db_instance.db_user}\",\"password\":\"${local.db_password}\",\"dbInstanceIdentifier\":\"${module.db_instance.db_identifier}\"}",
  "secretId": "${module.db_instance.db_secret_id}",
   "storageId": "${module.db_instance.db_identifier}",
   "forcedMod": "${var.forced_mod}"
}
JSON

    depends_on = [
    module.db_instance.db_secret_id,
  ]
}


output "result" {
  description = "String result of Lambda execution"
  value       = jsondecode(data.aws_lambda_invocation.credentials_manager.result)
}

In order to make sure that the RDS instance status is 'available' the lambda function also contains a waiter. When I manually execute the function everything works like a charm. But within in terraform it does not proceed from here:

data.aws_lambda_invocation.credentials_manager: Refreshing state...

However, when I look into AWS Cloud Watch I can see that the lambda function is being invoked by Terraform over and over again.

This is the lambda policy:

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "Stmt1589715377799",
      "Action": [
        "rds:ModifyDBInstance",
        "rds:DescribeDBInstances"
      ],
      "Effect": "Allow",
      "Resource": "*"
    }
  ]
}

The lambda function looks like this:

const secretsManager = require('aws-sdk/clients/secretsmanager')
const rds = require('aws-sdk/clients/rds')
const elastiCache = require('aws-sdk/clients/elasticache')
const log = require('loglevel')


/////////////////////////////////////////
// ENVIRONMENT VARIABLES
/////////////////////////////////////////
const logLevel = process.env["LOG_LEVEL"];
const region = process.env["REGION"]


/////////////////////////////////////////
// CONFIGURE LOGGER
log.setLevel(logLevel);
let protocol = []

/////////////////////////////////////////


/////////////////////////////////////////
// DEFINE THE CLIENTS
const SM = new secretsManager({ region })
const RDS = new rds({ region })
const ELC = new elastiCache({region})
/////////////////////////////////////////


/////////////////////////////////////////
// FUNCTION DEFINITIONS
/////////////////////////////////////////


// HELPERS

/**
 * @function waitForSeconds
 * Set a custom waiter.
 * 
 * @param {int} milseconds      - the milliseconds to set as timeout.
 * 
 */

const waitForSeconds = (ms) => {
   return new Promise(resolve => setTimeout(resolve, ms))
}




// AWS SECRETS MANAGER FUNCTIONS

/**
 * @function UpdateSecretInSM
 * The function updates the secrect value in the corresponding secret.
 * 
 * @param {string} secretId      - The id of the secret located in AWS SecretsManager 
 * @param {string} secretString  - The value of the new secret
 * 
 */
const UpdateSecretInSM = async (secretId,secretString) => {

    const params = {SecretId: secretId, SecretString: secretString}



    try {
        const data = await SM.updateSecret(params).promise()
        log.info(`[INFO]: Password for ${secretId} successfully changed in Scecrets Manager!`)
        let success = {Timestamp: new Date().toISOString(),Func: 'UpdateSecretInSM', Message: `Secret for ${secretId} successfully changed!`}
        protocol.push(success)
        return
    } catch (err) {
        log.debug("[DEBUG]: Error: ", err.stack);
        let error = {Timestamp: new Date().toISOString(),Func: 'UpdateSecretInSM', Error: err.stack}
        protocol.push(error)
        return
    }
}





/**
 * @function GetSecretFromSM
 * The function retrieves the specified secret from AWS SecretsManager.
 * Returns the password.
 * 
 * @param {string} secretId   - secretId that is available in AWS SecretsManager
 * 
 */
const GetSecretFromSM = async (secretId) => {


    try {
        const data = await SM.getSecretValue({SecretId: secretId}).promise()
        log.debug("[DEBUG]: Secret: ", data);
        let success = {Timestamp: new Date().toISOString(),Func: 'GetSecretFromSM', Message: 'Secret from SecretsManager successfully received!'}
        protocol.push(success)
        const { SecretString } = data
        const password = JSON.parse(SecretString)
        return password.password

    } catch (err) {

        log.debug("[DEBUG]: Error: ", err.stack);
        let error = {Timestamp: new Date().toISOString(),Func: 'GetSecretFromSM', Error: err.stack}
        protocol.push(error)
        return
    }

}


// AWS RDS FUNCTIONS

/**
 * @function ChangeRDSSecret
 * Change the secret of the specified RDS instance.
 * 
 * @param {string} rdsId     - id of the RDS instance
 * @param {string} password  - new password
 * 
 */
const ChangeRDSSecret = async (rdsId,password) => {

    const params = {
        DBInstanceIdentifier: rdsId,
        MasterUserPassword: password
    }



    try {
        await RDS.modifyDBInstance(params).promise() 

        log.info(`[INFO]: Password for ${rdsId} successfully changed!`)
        let success = {Timestamp: new Date().toISOString(), Func: 'ChangeRDSSecret', Message: `Secret for ${rdsId} successfully changed!`}
        protocol.push(success)
        return
    } catch (err) {

        log.debug("[DEBUG]: Error: ", err.stack);
        let error = {Timestamp: new Date().toISOString(), Func: 'ChangeRDSSecret', Error: err.stack}
        protocol.push(error)
        return 

    }

}


const DescribeRDSInstance = async(id) => {


    const params = { DBInstanceIdentifier : id }

    const secondsToWait = 10000

    try {
        let pendingModifications = true

        while (pendingModifications == true) {
            log.info(`[INFO]: Checking modified values for ${id}`)

            let data = await RDS.describeDBInstances(params).promise()
            console.log(data)

            // Extract the 'PendingModifiedValues' object
            let myInstance = data['DBInstances']
            myInstance = myInstance[0]

            if (myInstance.DBInstanceStatus === "resetting-master-credentials") {
                log.info(`[INFO]:Password change is being processed!`)
                 pendingModifications = false

            }

            log.info(`[INFO]: Waiting for ${secondsToWait/1000} seconds!`)
            await waitForSeconds(secondsToWait)
        }


        let success = {Timestamp: new Date().toISOString(), Func: 'DescribeRDSInstance', Message: `${id} available again!`}
        protocol.push(success)
        return 

    } catch (err) {
         log.debug("[DEBUG]: Error: ", err.stack);
        let error = {Timestamp: new Date().toISOString(), Func: 'DescribeRDSInstance', Error: err.stack}
        protocol.push(error)
        return 

    }

}


const WaitRDSForAvailableState = async(id) => {


/**
 * @function WaitRDSForAvailableState
 * Wait for the instance to be available again.
 * 
 * @param {string} id           - id of the RDS instance
 *
 */
    const params = { DBInstanceIdentifier: id}


    try {
        log.info(`[INFO]: Waiting for ${id} to be available again!`)
        const data = await RDS.waitFor('dBInstanceAvailable', params).promise()

        log.info(`[INFO]: ${id} available again!`)
        let success = {Timestamp: new Date().toISOString(), Func: 'WaitRDSForAvailableState', Message: `${id} available again!`}
        protocol.push(success)
        return
    } catch (err) {
        log.debug("[DEBUG]: Error: ", err.stack);
        let error = {Timestamp: new Date().toISOString(), Func: 'WaitRDSForAvailableState', Error: err.stack}
        protocol.push(error)
        return 

    }


}


// AWS ELASTICACHE FUNCTIONS

// ... removed since they follow the same principle like RDS





/////////////////////////////////////////
// Lambda Handler
/////////////////////////////////////////


exports.handler = async (event,context,callback) => {

    protocol = []
    log.debug("[DEBUG]: Event:", event)
    log.debug("[DEBUG]: Context:", context)

    // Variable for the final message the lambda function will return
    let finalValue



    // Get the password and rds from terraform output
    const secretString = event.secretString // manual input
    const secretId = event.secretId // coming from secretesmanager
    const storageId = event.storageId // coming from db identifier
    const forcedMod = event.forcedMod // manual input


    // Extract the password from the passed secretString to for comparison
    const passedSecretStringJSON = JSON.parse(secretString)
    const passedSecretString = passedSecretStringJSON.password


    const currentSecret = await GetSecretFromSM(secretId)

    // Case if the password has already been updated
    if (currentSecret !== "ChangeMeViaScriptOrConsole" && passedSecretString === "ChangeMeViaScriptOrConsole") {
        log.debug("[DEBUG]: No change necessary.")
        finalValue = {timestamp: new Date().toISOString(), 
            message: 'Lambda function execution finished!', 
            summary: 'Password already updated. It is not "ChangeMeViaScriptOrConsole."'}

        return finalValue
    }   

    // Case if the a new password has not been set yet
    if (currentSecret === "ChangeMeViaScriptOrConsole" && passedSecretString === "ChangeMeViaScriptOrConsole") {
        finalValue = {timestamp: new Date().toISOString(), 
            message: 'Lambda function execution finished!', 
            summary: 'Password still "ChangeMeViaScriptOrConsole". Please change me!'}

        return finalValue
    }

//   Case if the passed password is equal to the stored password and if pw modification is enforced
    if (currentSecret === passedSecretString && forcedMod === "no") {
        finalValue = {timestamp: new Date().toISOString(), 
            message: 'Lambda function execution finished!', 
            summary: 'Stored password is the same as the passed one. No changes made!'}

        return finalValue
    }


    // Case for changing the password
    if (passedSecretString !== "ChangeMeViaScriptOrConsole") {

        // Update the secret in SM for the specified RDS Instances
        await UpdateSecretInSM(secretId,secretString)

        log.debug("[DEBUG]: Secret updated for: ", secretId)

        // Change the new secret vom SM
        const updatedSecret = await GetSecretFromSM(secretId)

        log.debug("[DEBUG]: Updated secret: ", updatedSecret)


        if (secretId.includes("rds")) {

            // Update RDS instance with new secret and wait for it to be available again
            await ChangeRDSSecret(storageId, updatedSecret)
            await DescribeRDSInstance(storageId)
            await WaitRDSForAvailableState(storageId)

        } else if (secretId.includes("elasticache")) {

          // ... removed since it is analogeous to RDS


        } else {

            protocol.push(`No corresponding Storage Id exists for ${secretId}. Please check the Secret Id/Name in the terraform configuration.`)
        }


        finalValue ={timestamp: new Date().toISOString(), 
            message: 'Lambda function execution finished!', 
            summary: protocol}

        return finalValue

    } else {

        finalValue = {timestamp: new Date().toISOString(), 
            message: 'Lambda function execution finished!', 
            summary: 'Nothing changed'}

        return finalValue
    }




}

Anyone an idea how to solve or mitigate this behaviour?

Upvotes: 0

Views: 1507

Answers (1)

sogyals429
sogyals429

Reputation: 375

Can you please show the iam policy for your lambda function? By my understanding you might be missing this resource aws_lambda_permission for your lambda function. https://www.terraform.io/docs/providers/aws/r/lambda_permission.html

Upvotes: 1

Related Questions