CODEWITHSUNDEEP
node.jsamazon-dynamodb
Dave Stein
Dave Stein

Reputation: 9316

How do I batch delete with DynamoDB?

I am getting an error that "The provided key element does not match the schema". uuid is my primary partition key. I also have a primary sort key for version. I figured I can use batchWrite (docs) to delete all items with same uuid.

My ES6 code is as follows:

delete(uuid) {
  const promise = new Promise();
  const params = {
    RequestItems: {
      [this.TABLE]: [
        {
          DeleteRequest: {
            Key: { uuid: uuid }
          }
        }
      ]
    }
  };


  // this._client references the DocumentClient
  this._client.batchWrite(params, function(err, data) {
    if (err) {
      // this gets hit with error
      console.log(err);
      return promise.reject(err);
    }

    console.log(result);
    return promise.resolve(result);
  });

  return promise;
}

Not sure why it is erroring on the key that is the primary. I have seen posts about needing other indexes for times when I am searching by something that isn't a key. But I don't believe that's the case here.

Upvotes: 28

Views: 65522

Answers (6)

Mike Bendorf
Mike Bendorf

Reputation: 470

This is doable with Node lambda, but there are a few things you need to consider to address concurrency while processing large databases:

Here a snipped that I wrote:

const AWS = require("aws-sdk");
const dynamodb = new AWS.DynamoDB.DocumentClient();
const log = console.log;

exports.handler = async (event) => {

  log(event);
  let TableName = event.tableName;
  let params = {
  let TableName,
        FilterExpression: "userId = :uid",
        ExpressionAttributeValues: {
          ":uid": event.userId,
        },
      };
  let getItems = async (lastKey, items) => {
        if (lastKey) params.ExclusiveStartKey = lastKey;
        let resp = await dynamodb.scan(params).promise();
        let items = resp.Items.length
               ? items.concat(resp.Items.map((x) => x.id))
               : items;
        if (resp.LastEvaluatedKey)
          return await getItems(resp.LastEvaluatedKey, items);
        else return items;
      };
  let ids = await getItems(null, []);
  let idGroups = [];

  for (let i = 0; i < ids.length; i += 25) {
    idGroups.push(ids.slice(i, i + 25));
  }

  for (const gs of idGroups) {
    let delReqs = [];
    for (let id of gs) {
      delReqs.push({ DeleteRequest: { Key: { id } } });
    }
    let RequestItems = {};
    RequestItems[TableName] = delReqs;
    let d = await dynamodb
      .batchWrite({ RequestItems })
      .promise().catch((e) => log(e));
  }
  log(ids.length + " items processed");
  return {};
};

Upvotes: 6

Akhil Ghatiki
Akhil Ghatiki

Reputation: 1208

Why not use PartiQL. This approach is much more readable. (This too has a limit of 25 items per request just like BatchWriteITems)

// Import required AWS SDK clients and commands for Node.js.
import { BatchExecuteStatementCommand } from "@aws-sdk/client-dynamodb";
import { ddbDocClient } from "../libs/ddbDocClient.js";

const tableName = process.argv[2];
const movieYear1 = process.argv[3];
const movieTitle1 = process.argv[4];
const movieYear2 = process.argv[5];
const movieTitle2 = process.argv[6];

export const run = async (
  tableName,
  movieYear1,
  movieTitle1,
  movieYear2,
  movieTitle2
) => {
  try {
    const params = {
      Statements: [
        {
          Statement: "DELETE FROM " + tableName + "  where year=? and title=?",
          Parameters: [{ N: movieYear1 }, { S: movieTitle1 }],
        },
        {
          Statement: "DELETE FROM " + tableName + "  where year=? and title=?",
          Parameters: [{ N: movieYear2 }, { S: movieTitle2 }],
        },
      ],
    };
    const data = await ddbDocClient.send(
      new BatchExecuteStatementCommand(params)
    );
    console.log("Success. Items deleted.", data);
    return "Run successfully"; // For unit tests.
  } catch (err) {
    console.error(err);
  }
};
run(tableName, movieYear1, movieTitle1, movieYear2, movieTitle2);

Upvotes: 0

Alpa Sharma
Alpa Sharma

Reputation: 11

For batch delete, we can use batchWrite with DeleteRequest. Here is an example, in this, we are providing tableName whose data is to be deleted, the payload is an array of ids which we need to remove. In single request 25 items can be deleted. 

const AWS = require('aws-sdk');
const dynamodb= new AWS.DynamoDB.DocumentClient({ apiVersion: '2012-08-10' });

const tableName = "PlayerData";
const payload = [{id=101}, {id=105}, {id=106}];

    const deleteBatchData = async (tableName, payload, dynamodb) => {
        try {
            await dynamodb.batchWrite({
                RequestItems: {
                    [tableName]: payload.map(item => {
                        return {
                            DeleteRequest: {
                                Key: {
                                    id: item.id
                                }
                            }
                        };
                    })
                }
            }).
            promise().
            then((response) => {
                    return response;
                })
                .catch((err) => {
                    console.log("err ::", JSON.stringify(err))
                });
        } catch (err) {
            console.log('Error in deleteBatchData ', err);
        }
    }

Upvotes: 1

NubPro
NubPro

Reputation: 515

Not sure why nobody provided a proper answer.

Here's a lambda I did in nodeJS. It will perform a full scan on the table, then batch delete every 25 items per request.

Remember to change TABLE_NAME.

const AWS = require('aws-sdk');

const docClient = new AWS.DynamoDB.DocumentClient({ apiVersion: '2012-08-10' });

//const { TABLE_NAME } = process.env;
TABLE_NAME = "CHANGE ME PLEASE"

exports.handler = async (event) => {
    let params = {
        TableName: TABLE_NAME,
    };

    let items = [];
    let data = await docClient.scan(params).promise();
    items = [...items, ...data.Items];

    while (typeof data.LastEvaluatedKey != 'undefined') {
        params.ExclusiveStartKey = data.LastEvaluatedKey;

        data = await docClient.scan(params).promise();
        items = [...items, ...data.Items];
    }

    let leftItems = items.length;
    let group = [];
    let groupNumber = 0;

    console.log('Total items to be deleted', leftItems);

    for (const i of items) {
        const deleteReq = {
            DeleteRequest: {
                Key: {
                    id: i.id,
                },
            },
        };

        group.push(deleteReq);
        leftItems--;

        if (group.length === 25 || leftItems < 1) {
            groupNumber++;

            console.log(`Batch ${groupNumber} to be deleted.`);

            const params = {
                RequestItems: {
                    [TABLE_NAME]: group,
                },
            };

            await docClient.batchWrite(params).promise();

            console.log(
                `Batch ${groupNumber} processed. Left items: ${leftItems}`
            );

            // reset
            group = [];
        }
    }

    const response = {
        statusCode: 200,
        //  Uncomment below to enable CORS requests
        //  headers: {
        //      "Access-Control-Allow-Origin": "*"
        //  },
        body: JSON.stringify('Hello from Lambda!'),
    };
    return response;
};

Upvotes: 4

wzr1337
wzr1337

Reputation: 3767

Be aware, that you need to follow instructions:

src: https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html

DeleteRequest - Perform a DeleteItem operation on the specified item. The item to be deleted is identified by a Key subelement:

Key - A map of primary key attribute values that uniquely identify the item. Each entry in this map consists of an attribute name and an attribute value. For each primary key, you must provide all of the key attributes. For example, with a simple primary key, you only need to provide a value for the partition key. For a composite primary key, you must provide values for both the partition key and the sort key.

Upvotes: 1

notionquest
notionquest

Reputation: 39186

Here is the batch write delete request sample. This code has been tested and working fine. If you change this code for your requirement, it should work.

Table Definition:-

Bag - Table Name

bag - Hash Key

No partition key in 'Bag' table

Batch Write Code:-

var AWS = require("aws-sdk");

AWS.config.update({
    region : "us-west-2",
    endpoint : "http://localhost:8000"
});

var documentclient = new AWS.DynamoDB.DocumentClient();

var itemsArray = [];

var item1 = {
    DeleteRequest : {
        Key : {
            'bag' : 'b1'    
        }
    }
};

itemsArray.push(item1);

var item2 = {
    DeleteRequest : {
        Key : {
            'bag' : 'b2'    
        }
    }
};

itemsArray.push(item2);

var params = {
    RequestItems : {
        'Bag' : itemsArray
    }
};
documentclient.batchWrite(params, function(err, data) {
    if (err) {
        console.log('Batch delete unsuccessful ...');
        console.log(err, err.stack); // an error occurred
    } else {
        console.log('Batch delete successful ...');
        console.log(data); // successful response
    }

});

Output:-

Batch delete successful ...
{ UnprocessedItems: {} }

Upvotes: 37

Related Questions