Django S3 Bucket File Upload

Question

Trying to upload a file to S3 Bucket, but I am receiving the following error:

InvalidRequestThe authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.18B2904E53096953+pm4kJtsnkRDaopEeQ0JxAVcyhh9stYsTR4tnTkKMOm/HQv3N0ZAdrbZ42H1ggbB8CIOBRISwEs=

I believe my problem is that the code is using the old v2 S3 signature process and I need to update the signature code to the v4 S3 format.

s3_config.py

AWS_UPLOAD_BUCKET = 'upload-bucket-s4-frankfurt'

AWS_UPLOAD_USERNAME = 'upload-user'

AWS_UPLOAD_GROUP = 'CFE_group'

AWS_UPLOAD_REGION = 'eu-central-1'

AWS_UPLOAD_ACCESS_KEY_ID = 'removed'

AWS_UPLOAD_SECRET_KEY = 'removed'

AWS_S3_SIGNATURE_VERSION = 's3v4

'

Views.py

import base64
import hashlib
import hmac
import os
import time
import datetime
from django.utils import timezone
from rest_framework import permissions, status, authentication
from rest_framework.response import Response
from rest_framework.views import APIView
from .config_s3_aws import (
    AWS_UPLOAD_BUCKET,
    AWS_UPLOAD_REGION,
    AWS_UPLOAD_ACCESS_KEY_ID,
    AWS_UPLOAD_SECRET_KEY,
)
from .models import FileItem
import boto3
from botocore.client import Config
# Get the service client with sigv4 configured
s3 = boto3.client('s3', config=Config(signature_version='s3v4'))



class FilePolicyAPI(APIView):

permission_classes = [permissions.IsAuthenticated]
authentication_classes = [authentication.SessionAuthentication]

def post(self, request, *args, **kwargs):

    filename_req = request.data.get('filename')
    if not filename_req:
            return Response({"message": "A filename is required"}, status=status.HTTP_400_BAD_REQUEST)
    policy_expires = int(time.time()+1000)



    user = request.user
    username_str = str(request.user.username)

    file_obj = FileItem.objects.create(user=user, name=filename_req)
    file_obj_id = file_obj.id
    upload_start_path = "{username}/{file_obj_id}/".format(
                username = username_str,
                file_obj_id=file_obj_id
        )
    _, file_extension = os.path.splitext(filename_req)
    filename_final = "{file_obj_id}{file_extension}".format(
                file_obj_id= file_obj_id,
                file_extension=file_extension

            )

    final_upload_path = "{upload_start_path}{filename_final}".format(
                             upload_start_path=upload_start_path,
                             filename_final=filename_final,
                        )
    if filename_req and file_extension:
        """
        Save the eventual path to the Django-stored FileItem instance
        """
        file_obj.path = final_upload_path
        file_obj.save()

    policy_document_context = {
        "expire": policy_expires,
        "bucket_name": AWS_UPLOAD_BUCKET,
        "key_name": "",
        "acl_name": "private",
        "content_name": "",
        "content_length": 524288000,
        "upload_start_path": upload_start_path,

        }
    policy_document = """
    {"expiration": "2019-01-01T00:00:00Z",
      "conditions": [
        {"bucket": "%(bucket_name)s"},
        ["starts-with", "$key", "%(upload_start_path)s"],
        {"acl": "%(acl_name)s"},

        ["starts-with", "$Content-Type", "%(content_name)s"],
        ["starts-with", "$filename", ""],
        ["content-length-range", 0, %(content_length)d]
      ]
    }
    """ % policy_document_context
    aws_secret = str.encode(AWS_UPLOAD_SECRET_KEY)
    policy_document_str_encoded = str.encode(policy_document.replace(" ", ""))
    url = 'https://{bucket}.s3-{region}.amazonaws.com/'.format(
                    bucket=AWS_UPLOAD_BUCKET,
                    region=AWS_UPLOAD_REGION
                    )
    policy = base64.b64encode(policy_document_str_encoded)
    signature = base64.b64encode(hmac.new(aws_secret, policy, hashlib.sha1).digest())
    data = {
        "policy": policy,
        "signature": signature,
        "key": AWS_UPLOAD_ACCESS_KEY_ID,
        "file_bucket_path": upload_start_path,
        "file_id": file_obj_id,
        "filename": filename_final,
        "url": url,
        "username": username_str,
    }
    return Response(data, status=status.HTTP_200_OK)

Answer 1

Don't forget to import Config. Also If you have your own config class, then change its name.

from botocore.client import Config

s3 = boto3.client('s3',config=Config(signature_version='s3v4'),region_name=app.config["AWS_REGION"],aws_access_key_id=app.config['AWS_ACCESS_KEY'], aws_secret_access_key=app.config['AWS_SECRET_KEY'])
s3.upload_fileobj(file,app.config["AWS_BUCKET_NAME"],file.filename)
url = s3.generate_presigned_url('get_object', Params = {'Bucket':app.config["AWS_BUCKET_NAME"] , 'Key': file.filename}, ExpiresIn = 10000)

Answer 2

In your code, you configured the S3 client but you don't seem to be using it.

Instead of generating the signature and url in your code, you can use boto3's generate_presigned_url method like this:

s3 = boto3.client('s3')
bucket_params = {'Key': key, 'Bucket': self.BUCKET_NAME, 'ACL': 'public-read'}
url = s3.generate_presigned_url(ClientMethod='put_object', Params=bucket_params)

The code above will return a URL that can be used to upload a file like this:

curl -X PUT -H 'x-amz-acl: public-read'  -T icon_1.png -L 'URL HERE'

Remember to configure the ACL (Access Control List) to your desired settings.

Check out the documentation for further information:

https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html?highlight=presigned#S3.Client.generate_presigned_url

https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-presigned-urls.html