How to force delete all versions of objects in S3 bucket and then eventually delete the entire bucket using aws-sdk-go?

Question

I have an S3 bucket with versioning enabled. The bucket has few files which have versions. I have written a sample golang program which can do the following:

• GetBucketVersioning - It is able to get bucket versioning status i.e., Enabled
• ListObjects - It is able to list the bucket objects
• DeleteObjects - It is able to delete the bucket objects (but it just adds 'Delete Marker' only to the latest version of each object. The version history of the objects still remain undeleted)
• DeleteBucket: This operation fails with the error message:

"BucketNotEmpty: The bucket you tried to delete is not empty.
You must delete all versions in the bucket."

Could you advise how to force delete ALL VERSIONS of ALL OBJECTS in an S3 bucket so that I can ultimately delete the entire bucket, using aws-sdk-go, please?

Answer 1

According to the docs, DeleteBucket states,

All objects (including all object versions and delete markers) in the bucket must be deleted before the bucket itself can be deleted.

Now, to delete the versions from a versioning-enabled bucket, we can

1. use DeleteObject, which states,

To remove a specific version, you must be the bucket owner and you must use the version Id subresource. Using this subresource permanently deletes the version.

2. use DeleteObjects, which similarly states,

In the XML, you provide the object key names, and optionally, version IDs if you want to delete a specific version of the object from a versioning-enabled bucket.

I put together a sample program, that I tested against LocalStack, after using the following commands(prerequisites - Docker, Docker Compose, AWS CLI) to create a bucket and populate it with files to include versions.

curl -O https://raw.githubusercontent.com/localstack/localstack/master/docker-compose.yml
export SERVICES="s3"
docker-compose up

export AWS_ACCESS_KEY_ID="test"
export AWS_SECRET_ACCESS_KEY="test"
export AWS_DEFAULT_REGION="us-east-1"
aws --endpoint-url=http://localhost:4566 s3 mb s3://testbucket
aws --endpoint-url=http://localhost:4566 s3api put-bucket-versioning --bucket testbucket --versioning-configuration Status=Enabled
for i in 1 2 3; do
    aws --endpoint-url=http://localhost:4566 s3 cp main.go s3://testbucket/main.go
    aws --endpoint-url=http://localhost:4566 s3 cp go.mod s3://testbucket/go.mod
done

aws --endpoint-url=http://localhost:4566 s3api list-object-versions --bucket testbucket

Set the following environment variables before running it

export AWS_ENDPOINT="http://localhost:4566"
export S3_BUCKET="testbucket"

package main

import (
    "context"
    "fmt"
    "log"
    "os"

    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/config"
    "github.com/aws/aws-sdk-go-v2/service/s3"
    "github.com/aws/aws-sdk-go-v2/service/s3/types"
)

type s3Client struct {
    *s3.Client
}

func main() {
    awsEndpoint := os.Getenv("AWS_ENDPOINT")
    bucketName := os.Getenv("S3_BUCKET")

    cfg, err := config.LoadDefaultConfig(context.TODO(),
        config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
            func(service, region string, options ...interface{}) (aws.Endpoint, error) {
                return aws.Endpoint{
                    URL:               awsEndpoint,
                    HostnameImmutable: true,
                }, nil
            })),
    )
    if err != nil {
        log.Fatalf("Cannot load the AWS configs: %s", err)
    }

    serviceClient := s3.NewFromConfig(cfg)

    client := &s3Client{
        Client: serviceClient,
    }

    fmt.Printf(">>> Bucket: %s\n", bucketName)

    objects, err := client.listObjects(bucketName)
    if err != nil {
        log.Fatal(err)
    }
    if len(objects) > 0 {
        fmt.Printf(">>> List objects in the bucket: \n")
        for _, object := range objects {
            fmt.Printf("%s\n", object)
        }
    } else {
        fmt.Printf(">>> No objects in the bucket.\n")
    }

    if client.versioningEnabled(bucketName) {
        fmt.Printf(">>> Versioning is enabled.\n")
        objectVersions, err := client.listObjectVersions(bucketName)
        if err != nil {
            log.Fatal(err)
        }
        if len(objectVersions) > 0 {
            fmt.Printf(">>> List objects with versions: \n")
            for key, versions := range objectVersions {
                fmt.Printf("%s: ", key)
                for _, version := range versions {
                    fmt.Printf("\n\t%s ", version)
                }
                fmt.Println()
            }
        }

        if len(objectVersions) > 0 {
            fmt.Printf(">>> Delete objects with versions.\n")
            if err := client.deleteObjects(bucketName, objectVersions); err != nil {
                log.Fatal(err)
            }

            objectVersions, err = client.listObjectVersions(bucketName)
            if err != nil {
                log.Fatal(err)
            }
            if len(objectVersions) > 0 {
                fmt.Printf(">>> List objects with versions after deletion: \n")
                for key, version := range objectVersions {
                    fmt.Printf("%s: %s\n", key, version)
                }
            } else {
                fmt.Printf(">>> No objects in the bucket after deletion.\n")
            }
        }
    }

    fmt.Printf(">>> Delete the bucket.\n")
    if err := client.deleteBucket(bucketName); err != nil {
        log.Fatal(err)
    }

}

func (c *s3Client) versioningEnabled(bucket string) bool {
    output, err := c.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
        Bucket: aws.String(bucket),
    })
    if err != nil {
        return false
    }
    return output.Status == "Enabled"
}

func (c *s3Client) listObjects(bucket string) ([]string, error) {
    var objects []string
    output, err := c.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
        Bucket: aws.String(bucket),
    })
    if err != nil {
        return nil, err
    }

    for _, object := range output.Contents {
        objects = append(objects, aws.ToString(object.Key))
    }

    return objects, nil
}

func (c *s3Client) listObjectVersions(bucket string) (map[string][]string, error) {
    var objectVersions = make(map[string][]string)
    output, err := c.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
        Bucket: aws.String(bucket),
    })
    if err != nil {
        return nil, err
    }

    for _, object := range output.Versions {
        if _, ok := objectVersions[aws.ToString(object.Key)]; ok {
            objectVersions[aws.ToString(object.Key)] = append(objectVersions[aws.ToString(object.Key)], aws.ToString(object.VersionId))
        } else {
            objectVersions[aws.ToString(object.Key)] = []string{aws.ToString(object.VersionId)}
        }
    }

    return objectVersions, err
}

func (c *s3Client) deleteObjects(bucket string, objectVersions map[string][]string) error {
    var identifiers []types.ObjectIdentifier
    for key, versions := range objectVersions {
        for _, version := range versions {
            identifiers = append(identifiers, types.ObjectIdentifier{
                Key:       aws.String(key),
                VersionId: aws.String(version),
            })
        }
    }

    _, err := c.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
        Bucket: aws.String(bucket),
        Delete: &types.Delete{
            Objects: identifiers,
        },
    })
    if err != nil {
        return err
    }
    return nil
}

func (c *s3Client) deleteBucket(bucket string) error {
    _, err := c.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
        Bucket: aws.String(bucket),
    })
    if err != nil {
        return err
    }

    return nil
}

Note in the listObjectVersions method, I am mapping the VersionIds with the Keys.

    for _, object := range output.Versions {
        if _, ok := objectVersions[aws.ToString(object.Key)]; ok {
            objectVersions[aws.ToString(object.Key)] = append(objectVersions[aws.ToString(object.Key)], aws.ToString(object.VersionId))
        } else {
            objectVersions[aws.ToString(object.Key)] = []string{aws.ToString(object.VersionId)}
        }
    }

Then in the deleteObjects method, when passing the ObjectIdentifiers, I pass the Key and the ObjectIds for all the versions of an object.

    for key, versions := range objectVersions {
        for _, version := range versions {
            identifiers = append(identifiers, types.ObjectIdentifier{
                Key:       aws.String(key),
                VersionId: aws.String(version),
            })
        }
    }

Answer 2

This seem to be impossible using the golang sdk. They didn't implement a delete version function.