Download Objects from S3 Bucket using c#

Question

Im trying to download object from S3 bucket facing below issue The Security token included in the request is Invalid . Please check and correct where is the mistake.

Below is my code 1. Get Temporary credentails:

main()    
{
    string path = "http://XXX.XXX.XXX./latest/meta-data/iam/security-credentials/EC2_WLMA_Permissions";

                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(path);
                request.Method = "GET";
                request.ContentType = "application/json";
                HttpWebResponse response = request.GetResponse() as HttpWebResponse;
                string result = string.Empty;
                using (StreamReader reader = new StreamReader(response.GetResponseStream()))
                {
                    result = reader.ReadToEnd();
                    dynamic metaData = JsonConvert.DeserializeObject(result);
                    _awsAccessKeyId = metaData.AccessKeyId;
                    _awsSecretAccessKey = metaData.SecretAccessKey;
                }
}

2. Create SessionAWSCredentials instance:

   SessionAWSCredentials tempCredentials =
                    GetTemporaryCredentials(_awsAccessKeyId, _awsSecretAccessKey);
   

   //GetTemporaryCredentials method:

      private static SessionAWSCredentials GetTemporaryCredentials(
                       string accessKeyId, string secretAccessKeyId)
       {             
   
           AmazonSecurityTokenServiceClient stsClient =
           new AmazonSecurityTokenServiceClient(accessKeyId,
                                                    secretAccessKeyId);
           Console.WriteLine(stsClient.ToString());
           GetSessionTokenRequest getSessionTokenRequest =
                                            new GetSessionTokenRequest();
   
           getSessionTokenRequest.DurationSeconds = 7200; // seconds
           GetSessionTokenResponse sessionTokenResponse =
                         stsClient.GetSessionToken(getSessionTokenRequest);
   
           Console.WriteLine(sessionTokenResponse.ToString());
           Credentials credentials = sessionTokenResponse.Credentials;
           Console.WriteLine(credentials.ToString());
   
           SessionAWSCredentials sessionCredentials =
               new SessionAWSCredentials(credentials.AccessKeyId,
                                         credentials.SecretAccessKey,
                                         credentials.SessionToken);
   
   
           return sessionCredentials;
       }
   

3. Get files from S3 using AmazonS3Client:

   using (IAmazonS3 client = new AmazonS3Client(tempCredentials,RegionEndpoint.USEast1))                        
       {
                       GetObjectRequest request = new GetObjectRequest();
                       request.BucketName = "bucketName" + @"/" + "foldername";
                       request.Key = "Terms.docx";
                       GetObjectResponse response = client.GetObject(request);
                       response.WriteResponseStreamToFile("C:\\MyFile.docx");
       }
   

Answer 1

Here is what I have done to download the files from S3 bucket,

var AwsImportFilePathParcel = "TEST/TEMP"

IAmazonS3 client = new AmazonS3Client(AwsAccessKey,AwsSecretKey);
            
S3DirectoryInfo info = new S3DirectoryInfo(client, S3BucketName, AwsImportFilePathParcel);
            
S3FileInfo[] s3Files = info.GetFiles(pattenForParcel);

now in s3Files, you have all the files which are on provided location, using for each you can save all files in to your system

foreach (var fileInfo in s3Files)
{

    var localPath = Path.Combine("C:\TEST\", fileInfo.Name);
    var file = fileInfo.CopyToLocal(localPath);
}

Answer 2

Following is how I download it. I need to download only .zip files in this case. Restricting to only required file types (.zip in my case), helped me to avoid errors related to (416) Requested Range Not Satisfiable

    public static class MyAWS_S3_Helper
    {
        static string _S3Key = System.Configuration.ConfigurationManager.ConnectionStrings["S3BucketKey"].ConnectionString;
        static string _S3SecretKey = System.Configuration.ConfigurationManager.ConnectionStrings["S3BucketSecretKey"].ConnectionString;

        public static void S3Download(string bucketName, string _ObjectKey, string downloadPath)
        {
            IAmazonS3 _client = new AmazonS3Client(_S3Key, _S3SecretKey, Amazon.RegionEndpoint.USEast1);
            TransferUtility fileTransferUtility = new TransferUtility(_client);
            fileTransferUtility.Download(downloadPath + "\\" + _ObjectKey, bucketName, _ObjectKey);
            _client.Dispose();
        }

        public static async Task AsyncDownload(string bucketName, string downloadPath, string requiredSunFolder)
        {
            var bucketRegion = RegionEndpoint.USEast1; //Change it
            var credentials = new BasicAWSCredentials(_S3Key, _S3SecretKey);
            var client = new AmazonS3Client(credentials, bucketRegion);

            var request = new ListObjectsV2Request
            {
                BucketName = bucketName,
                MaxKeys = 1000
            };

            var response = await client.ListObjectsV2Async(request);
            var utility = new TransferUtility(client);

            foreach (var obj in response.S3Objects)
            {
                string currentKey = obj.Key;
                double sizeCheck = Convert.ToDouble(obj.Size);
                int fileNameLength = currentKey.Length;
                Console.WriteLine(currentKey + "---" + fileNameLength.ToString());

                if (currentKey.Contains(requiredSunFolder))
                {
                    if (currentKey.Contains(".zip")) //This helps to avoid errors related to (416) Requested Range Not Satisfiable
                    {
                        try
                        {
                            S3Download(bucketName, currentKey, downloadPath);
                        }
                        catch (Exception exTest)
                        {
                            string messageTest = currentKey + "-" + exTest;
                        }
                    }
                }
            }

        }
    }

Here is how it is called

        static void Main(string[] args)
        {
            string downloadPath = @"C:\SourceFiles\TestDownload";
            Task  awsTask = MyAWS_S3_Helper.AsyncDownload("my-files", downloadPath, "mysubfolder");
            awsTask.Wait();
        }

Answer 3

Bucketname, Accesskey and secretkey, I took from web config. You could type manually.

 public void DownloadObject(string imagename)
    {
        RegionEndpoint bucketRegion = RegionEndpoint.USEast1;
        IAmazonS3 client = new AmazonS3Client(bucketRegion);

        string accessKey = System.Configuration.ConfigurationManager.AppSettings["AWSAccessKey"];
        string secretKey = System.Configuration.ConfigurationManager.AppSettings["AWSSecretKey"];            
        AmazonS3Client s3Client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey), Amazon.RegionEndpoint.USEast1);
        string objectKey = "EMR" + "/" + imagename;
        //EMR is folder name of the image inside the bucket 
        GetObjectRequest request = new GetObjectRequest();
        request.BucketName = System.Configuration.ConfigurationManager.AppSettings["bucketname"];      
        request.Key = objectKey;
        GetObjectResponse response = s3Client.GetObject(request);         
        response.WriteResponseStreamToFile("D:\\Test\\"+ imagename);

    }

//> D:\Test\ is local file path.

Answer 4

We do something a little simpler for interfacing with S3 (downloads and uploads)

It looks like you went the more complex approach. You should try just using the TransferUtility instead:

TransferUtility fileTransferUtility =
    new TransferUtility(
        new AmazonS3Client("ACCESS-KEY-ID", "SECRET-ACCESS-KEY", Amazon.RegionEndpoint.CACentral1));

// Note the 'fileName' is the 'key' of the object in S3 (which is usually just the file name)
fileTransferUtility.Download(filePath, "my-bucket-name", fileName);

---

NOTE: TransferUtility.Download() returns void because it downloads the file to the path specified in the filePath argument. This may be a little different than what you were expecting but you can still open a FileStream to that path afterwards and manipulate the file all you want. For example:

using (FileStream fileDownloaded = new FileStream(filePath, FileMode.Open, FileAccess.Read))
{
    // Do stuff with our newly downloaded file
}