Download folder from Amazon S3 bucket using .net SDK

Question

How to download entire folder present inside s3 bucket using .net sdk.Tried with below code, it throws invalid key.I need to download all files present inside nested pesudo folder present inside bucket and removing file download limitations to 1000 which is default.

public static void DownloadFile()
{
var client = new AmazonS3Client(keyId, keySecret, bucketRegion);
        ListObjectsV2Request request = new ListObjectsV2Request
        {
            BucketName = bucketName + "/private/TargetFolder",         
            MaxKeys = 1000
        };
        try
        {
            ListObjectsV2Response bucketResponse = client.ListObjectsV2(request);
            foreach (S3Object o in bucketResponse.S3Objects)
            {
                var getRequest = new GetObjectRequest
                {
                    BucketName = bucketResponse.Name + "/private/TargetFolder",
                    Key = bucketResponse.Name +"/private/TargetFolder/"+ o.Key
                };
                var response = client.GetObject(getRequest);
                response.WriteResponseStreamToFile(downloadLocation + "\\" + o.Key);
                var responseCode = response.HttpStatusCode;
                if (response.HttpStatusCode == System.Net.HttpStatusCode.OK)
                {
                    Console.WriteLine($"Success downloaded : {o.Key}");
                }
                else if (response.HttpStatusCode == System.Net.HttpStatusCode.RequestTimeout)
                {
                    Console.WriteLine("Request Timeout error.");
                }
                else if (response.HttpStatusCode == System.Net.HttpStatusCode.ServiceUnavailable)
                {
                    Console.WriteLine("Service Unavailable.");
                }
                else if (response.HttpStatusCode == System.Net.HttpStatusCode.InternalServerError)
                {
                    Console.WriteLine("Internal Server error.");
                }
                else
                {
                    Console.WriteLine("Please check the provided AWS Credentials.");
                }
            }
        }
        catch (AmazonS3Exception amazonS3Exception)
        {
            if (amazonS3Exception.ErrorCode != null &&
                (amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId") || amazonS3Exception.ErrorCode.Equals("InvalidSecurity")))
            {
                Console.WriteLine("Please check the provided AWS Credentials.");
            }
            else
            {
                Console.WriteLine(amazonS3Exception.Message);
            }

        }
        catch (Exception ex)
        {
            Console.WriteLine(ex.Message);
        }
        Console.ReadLine();
    }

Thanks in advance!

Answer 1

The solution marked as accepted uses client.ListObjects(request) which says on the descrition that will only take a max of 1000 files, if there are more files on the folder maybe the code below will work. Thanks.

    public void DownloadallBucketFiles(string bucketName, string folderPath = 
    null)
    {
         var bucketRegion = RegionEndpoint.USEast1;
         var credentials = new BasicAWSCredentials(S3CredentialsKey, 
         S3CredentialsSecret);
         var s3Client = new AmazonS3Client(credentials, bucketRegion);     

         var utility = new TransferUtility(s3Client);
         var dir = new S3DirectoryInfo(s3Client, bucketName);
         var filesInBucket = dir.EnumerateFiles()?.ToList();
         var path = (string.IsNullOrEmpty(folderPath)) ? S3LocalTempDir : 
         folderPath;

         if (filesInBucket == null)
             return;

         filesInBucket.ForEach(file =>
         {
             try
             {
                  if (!File.Exists(Path.Combine(path, file.Name)))
                     utility.Download($"{path}\\{file.Name}", bucketName, 
                     file.Name);

                  Console.WriteLine(file.Name + " Processed");
             }
             catch(Exception ex)
             {
                  Console.WriteLine(file.Name + $" download failed with error: 
                  {ex.Message}");
             }
         });
     }
  }

Answer 2

Try this, it works for me

public static void DownloadFile()
        {
            System.Net.ServicePointManager.SecurityProtocol = System.Net.SecurityProtocolType.Tls12;
            ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls | SecurityProtocolType.Tls | SecurityProtocolType.Tls;

            var client = new AmazonS3Client(keyId, keySecret, bucketRegion);
            ListObjectsRequest request = new ListObjectsRequest();

            request.BucketName = "BUCKET_NAME";
            request.Prefix = "private/TargetFolder";
            request.Delimiter = "/";
            request.MaxKeys = 1000;

            ListObjectsResponse response = client.ListObjects(request);
            var x = response.S3Objects;

            foreach (var objt in x)
            {
                GetObjectRequest request1 = new GetObjectRequest();
                request1.BucketName = "BUCKET_NAME";
                request1.Key = objt.Key;

                GetObjectResponse Response = client.GetObject(request1);
if(objt.Size > 0){
                using (Stream responseStream = Response.ResponseStream)
                {
                    Response.WriteResponseStreamToFile(downloadLocation + "\\" + objt.Key);
                }
}
            }
        }

Answer 3

If you are always getting 0 in S3Objects.Count, try without using Delimiter property:

public async Task DownloadDirectoryAsync()
{
    var bucketRegion = RegionEndpoint.USEast2;
    var credentials = new BasicAWSCredentials(accessKey, secretKey);
    var client = new AmazonS3Client(credentials, bucketRegion);

    var bucketName = "bucketName";
    var request = new ListObjectsV2Request
    {
        BucketName = bucketName,
        Prefix = "directorey/",
        MaxKeys = 1000
    };

    var response = await client.ListObjectsV2Async(request);
    var utility = new TransferUtility(s3Client);
    var downloadPath = "c:\\your_folder";
    foreach (var obj in response.S3Objects)
    {
        utility.Download($"{downloadPath}\\{obj.Key}", bucketName, obj.Key);
    }
} 

And of course, you need s3:ListBucket permission