CloudFormation Custom Resource not finishing deleting

Question

I have setup my custom resource to return immediately on deletes

const aws = require('aws-sdk')
const util = require('util')

exports.handler = (event, context) => {
  console.log('Event>>>')
  console.log(JSON.stringify(event))
  aws.config.update({ region: event.ResourceProperties.Region })

  if (event.RequestType === 'Delete') return ApiMethodCustom.sendResponse(event, context, 'SUCCESS') // HERE!

  ApiMethodCustom.setupIntegration(event, context)
}

  static async sendResponse(event, context, responseStatus, responseData = {}) {
    var responseBody = JSON.stringify({
      Status: responseStatus,
      Reason: "See the details in CloudWatch Log Stream: " + context.logStreamName,
      PhysicalResourceId: context.logStreamName,
      StackId: event.StackId,
      RequestId: event.RequestId,
      LogicalResourceId: event.LogicalResourceId,
      Data: responseData
    });

    console.log("RESPONSE BODY:\n", responseBody);

    var https = require("https");
    var url = require("url");

    var parsedUrl = url.parse(event.ResponseURL);
    var options = {
      hostname: parsedUrl.hostname,
      port: 443,
      path: parsedUrl.path,
      method: "PUT",
      headers: {
        "content-type": "",
        "content-length": responseBody.length
      }
    };

    console.log("SENDING RESPONSE...\n");

    var request = https.request(options, function (response) {
      console.log("STATUS: " + response.statusCode);
      console.log("HEADERS: " + JSON.stringify(response.headers));
      // Tell AWS Lambda that the function execution is done
      context.done();
    });

    request.on("error", function (error) {
      console.log("sendResponse Error:" + error);
      // Tell AWS Lambda that the function execution is done
      context.done();
    });

    // write data to request body
    request.write(responseBody);
    request.end();
  }

But it appears that CloudFormation is stuck in DELETE_IN_PROGRESS. Why is that?

In my logs, it seems like Lambda finished execution correctly:

2018-09-09T01:52:06.913Z    f48808d0-b3d2-11e8-9e84-5b218cad3090
{
    "RequestType": "Delete",
    "ServiceToken": "arn:aws:lambda:ap-southeast-1:621567429603:function:income2-base-ApiVpcIntegration",
    "ResponseURL": "https://cloudformation-custom-resource-response-apsoutheast1.s3-ap-southeast-1.amazonaws.com/arn%3Aaws%3Acloudformation%3Aap-southeast-1%3A621567429603%3Astack/test/5a34d100-b370-11e8-b89d-503a138dba36%7CApiTestIntegration%7C979b1814-d94c-4a49-b9f7-2fa352ab88f5?AWSAccessKeyId=AKIAIKQZQ3QDXOJPHOPA&Expires=1536465125&Signature=O2O0entoTXHCYp5jbJehghtE9Ck%3D",
    "StackId": "arn:aws:cloudformation:ap-southeast-1:621567429603:stack/test/5a34d100-b370-11e8-b89d-503a138dba36",
    "RequestId": "979b1814-d94c-4a49-b9f7-2fa352ab88f5",
    "LogicalResourceId": "ApiTestIntegration",
    "PhysicalResourceId": "2018/09/08/[$LATEST]b8a3df0fca884fe3b8abdde3ab525ac0",
    "ResourceType": "Custom::ApiVpcIntegration",
    "ResourceProperties": {
        "ServiceToken": "arn:aws:lambda:ap-southeast-1:621567429603:function:income2-base-ApiVpcIntegration",
        "ConnectionId": "24lbti",
        "ResourceId": "x1gjyy",
        "RestApiId": "aaj0q4dbml",
        "Uri": "http://dropletapi-dev.2359media.net:3001/authentication",
        "HttpMethod": "GET"
    }
}

2018-09-09T01:52:06.914Z    f48808d0-b3d2-11e8-9e84-5b218cad3090    RESPONSE BODY:
{
    "Status": "SUCCESS",
    "Reason": "See the details in CloudWatch Log Stream: 2018/09/09/[$LATEST]29276598cb9c49c1b1da3672c8707c78",
    "PhysicalResourceId": "2018/09/09/[$LATEST]29276598cb9c49c1b1da3672c8707c78",
    "StackId": "arn:aws:cloudformation:ap-southeast-1:621567429603:stack/test/5a34d100-b370-11e8-b89d-503a138dba36",
    "RequestId": "979b1814-d94c-4a49-b9f7-2fa352ab88f5",
    "LogicalResourceId": "ApiTestIntegration",
    "Data": {}
}

Answer 1

I had a similar issue today while using the cfn-response package, which your code appears to be based on. The cfn-response package is based on a callback but your code also seems to partially use async/await (option with Runtime: node.js8.10).

In your case I suspect that you never saw the "STATUS: " or "HEADERS: " messages even if the response body was dumped to logs (synchronously). That mirrors my experience when using callback-based cfn-response mixed with async/await.

In other words, in all circumstances you will need to ensure that you send a response to Cloudformation (PUT to the event S3 ResponseURL) before your Lambda terminates or the template could hang for up to an hour before giving up and rolling back (probably with a Cloudformation error along the lines of "Failed to stabilise the resource...". Rollback (deletion) in turn can also take an hour because the delete also does not response appropriately. A bit more information here.

I ended up implementing custom resources much like this example on GitHub by https://github.com/rosberglinhares (MIT license) with a couple of differences; I didn't set-up a separate lambda to handle the sendResponse functionality and I made the custom resources server-less (using aws cloudformation package and aws cloudformation deploy commands).

Your ApiMethodCustom is not defined so it's hard for me to guide you on that implementation and so I am including my node.js8.10 code using async/await for reference.

First the Custom resource in the Cloudformation template:

---
AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31
Description: CustomResource Example Stack
Resources:
  CustomResource:
    Type: 'AWS::Serverless::Function'
    Properties:
      Runtime: nodejs8.10
      Handler: index.handler
      MemorySize: 128
      Timeout: 15
      Role: !GetAtt CustomResourceRole.Arn
      CodeUri: ./CustomResource/

  CustomResourceUser:
    Type: 'Custom::CustomResourceUser'
    Properties:
      ServiceToken: !GetAtt CustomResource.Arn
      ...

Note that CodeUri is relative to the template path. You will need to define the IAM role and policies for CustomResourceRole.

Now for the CustomResource/index.js Lambda (you will also need to run "npm install --save axios" in the CustomResource directory):

'use strict';
const AWS = require('aws-sdk');
const axios = require('axios');

exports.handler = async (event, context) => {
  try {

    switch (event.RequestType) {
      case 'Create':
        await ApiMethodCustom.create(...);
        break;
      case 'Update':
        await ApiMethodCustom.update(...);
        break;
      case 'Delete':
        await ApiMethodCustom.delete(...);
        break;
    }
    console.info('Success for request type ${event.RequestType}');
    await sendResponse(event, context, 'SUCCESS', { } );
  } catch (error) {
    console.error('Error for request type ${event.RequestType}: ', error);
    await sendResponse(event, context, 'FAILED', { } );
  }
}

async function sendResponse (event, context, responseStatus, responseData, physicalResourceId) {

  var reason = responseStatus == 'FAILED' ? ('See the details in CloudWatch Log Stream: ' + context.logStreamName) : undefined;

  var responseBody = JSON.stringify({
    StackId: event.StackId,
    RequestId: event.RequestId,
    Status: responseStatus,
    Reason: reason,
    PhysicalResourceId: physicalResourceId || context.logStreamName,
    LogicalResourceId: event.LogicalResourceId,
    Data: responseData
  });

  var responseOptions = {
    headers: {
      'Content-Type': '',
      'Content-Length': responseBody.length
    }
  };

  console.info('Response body:\n', responseBody);

  try {
    await axios.put(event.ResponseURL, responseBody, responseOptions);

    console.info('CloudFormationSendResponse Success');
  } catch (error) {
    console.error('CloudFormationSendResponse Error:');

    if (error.response) {
      console.error(error.response.data);
      console.error(error.response.status);
      console.error(error.response.headers);
    } else if (error.request) {
      console.error(error.request);
    } else {
      console.error('Error', error.message);
    }

    console.error(error.config);

    throw new Error('Could not send CloudFormation response');
  }
}

For more information on using callback vs. async with AWS Lambda's have a look here.

Finally, note the use of Axios. It's promise-based and therefore supports await instead of callbacks.