Signature error message when uploading file with fineuploader and django into S3

Question

UPDATE: please see at the bottom of this message. It is a python3 / hmac version issue.

I'm setting up a file upload system with fine uploader, S3 and django 1.11. I setup the urls, template and view but here are the (client) error messages I get when attempting to upload a file:

[Fine Uploader 5.15.0] POST request for 0 has failed - response code 400
[Fine Uploader 5.15.0] Error attempting to parse signature response: SyntaxError: JSON Parse error: Unexpected identifier "invalid"
[Fine Uploader 5.15.0] Received an empty or invalid response from the server!
[Fine Uploader 5.15.0] Policy signing failed. Received an empty or invalid response from the server!

Fine uploader requires in django settings :

AWS_CLIENT_SECRET_KEY = IAM user Secret access key
AWS_SERVER_PUBLIC_KEY = IAM user Access key ID
AWS_SERVER_SECRET_KEY = IAM user Secret access key

I have my Access key ID and Secret access key from the iam user I created and set them as shown above. AWS_CLIENT_SECRET_KEY = AWS_SERVER_SECRET_KEY = IAM user secret key. I'm not sure this is correct and it might well be the problem, but I have no clue what is the difference between AWS_CLIENT_SECRET_KEY and AWS_SERVER_SECRET_KEY and where to find them if one is not iam secret key.

And here is the code:

View:

from django.shortcuts import render
from django.conf import settings
from django.http import HttpResponse, HttpRequest
from django.shortcuts import render
from django.views.decorators.http import require_http_methods
from django.views.decorators.csrf import csrf_exempt

import base64, hmac, hashlib, json, sys


import boto
from boto.s3.connection import Key, S3Connection

boto.set_stream_logger( 'boto' )
S3 = S3Connection( settings.AWS_SERVER_PUBLIC_KEY, settings.AWS_SERVER_SECRET_KEY )


def video_create_form( request ):
    return render( request, 'video_create_form_view.html' )


@csrf_exempt
def success_redirect_endpoint( request ):
    """ This is where the upload will snd a POST request after the
    file has been stored in S3.
    """
    return make_response( 200 )


@csrf_exempt
def handle_s3( request ):
    """ View which handles all POST and DELETE requests sent by Fine Uploader
    S3. You will need to adjust these paths/conditions based on your setup.
    """
    if request.method == "POST":
        return handle_POST( request )
    elif request.method == "DELETE":
        return handle_DELETE( request )
    else:
        return HttpResponse( status = 405 )


def handle_POST( request ):
    """ Handle S3 uploader POST requests here. For files <=5MiB this is a simple
    request to sign the policy document. For files >5MiB this is a request
    to sign the headers to start a multipart encoded request.
    """
    if request.POST.get( 'success', None ):
        return make_response( 200 )
    else:
        request_payload = json.loads( request.body )
        headers = request_payload.get( 'headers', None )
        if headers:
            # The presence of the 'headers' property in the request payload
            # means this is a request to sign a REST/multipart request
            # and NOT a policy document
            response_data = sign_headers( headers )
        else:
            if not is_valid_policy( request_payload ):
                return make_response( 400, { 'invalid': True } )
            response_data = sign_policy_document( request_payload )
        response_payload = json.dumps( response_data )
        return make_response( 200, response_payload )


def handle_DELETE( request ):
    """ Handle file deletion requests. For this, we use the Amazon Python SDK,
    boto.
    """
    if boto:
        bucket_name = request.REQUEST.get( 'bucket' )
        key_name = request.REQUEST.get( 'key' )
        aws_bucket = S3.get_bucket( bucket_name, validate = False )
        aws_key = Key( aws_bucket, key_name )
        aws_key.delete()
        return make_response( 200 )
    else:
        return make_response( 500 )


def make_response( status = 200, content = None ):
    """ Construct an HTTP response. Fine Uploader expects 'application/json'.
    """
    response = HttpResponse()
    response.status_code = status
    response[ 'Content-Type' ] = "application/json"
    response.content = content
    return response


def is_valid_policy( policy_document ):
    """ Verify the policy document has not been tampered with client-side
    before sending it off.
    """
    # bucket = settings.AWS_EXPECTED_BUCKET
    # parsed_max_size = settings.AWS_MAX_SIZE
    bucket = ''
    parsed_max_size = 0

    for condition in policy_document[ 'conditions' ]:
        if isinstance( condition, list ) and condition[ 0 ] == 'content-length-range':
            parsed_max_size = condition[ 2 ]
        else:
            if condition.get( 'bucket', None ):
                bucket = condition[ 'bucket' ]

    return bucket == settings.AWS_EXPECTED_BUCKET and parsed_max_size == settings.AWS_MAX_SIZE


def sign_policy_document( policy_document ):
    """ Sign and return the policy doucument for a simple upload.
    http://aws.amazon.com/articles/1434/#signyours3postform
    """
    policy = base64.b64encode( json.dumps( policy_document ) )
    signature = base64.b64encode(
            hmac.new( settings.AWS_CLIENT_SECRET_KEY, policy, hashlib.sha1 ).digest() )
    return {
        'policy'   : policy,
        'signature': signature
    }


def sign_headers( headers ):
    """ Sign and return the headers for a chunked upload. """
    return {
        'signature': base64.b64encode(
                hmac.new( settings.AWS_CLIENT_SECRET_KEY, headers, hashlib.sha1 ).digest() )
    }

Template:

{% load static %}




    
    
    
    


    Fine Uploader Gallery UI

Urls (imported into may url file)

from django.conf.urls import url
from videos.controllers.video_create_controller import video_create_form, handle_s3, success_redirect_endpoint


urlpatterns = [
    url( r'^video-create-form/$', video_create_form, name = 'video_create_form' ),
    url( r'^s3/signature', handle_s3, name = "s3_signee" ),
    url( r'^s3/delete', handle_s3, name = 's3_delete' ),
    url( r'^s3/success', success_redirect_endpoint, name = "s3_succes_endpoint" )
]

Settings

# Amazon variables. Be wary and don't hard-code your secret keys here. Rather,
# set them as environment variables, or read them from a file somehow.
AWS_CLIENT_SECRET_KEY = 'WDq/cy*****'
AWS_SERVER_PUBLIC_KEY = 'AK*****'
AWS_SERVER_SECRET_KEY = 'WDq/cy*****'

AWS_EXPECTED_BUCKET = 'mybucketname'
AWS_MAX_SIZE = 15000000

Cors policies

It doesn't appear to be aws side setting problem, as i'm able to get files into my bucket by other means.




    *
    GET
    HEAD
    PUT
    POST
    3000
    ETag
    *

UPDATE: It seems to be a problem with python version: after reproducing the installation with FLASK, I was able to get this error message concerning hmac:

 raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__)
TypeError: key: expected bytes or bytearray, but got 'str

Setting a venv with python 2.7 fixed the issue and I got all the process working. I'm investigating the issue, if someone has a fix please tell.

'

Robert Brax · Accepted Answer

Here is the view code, fully updated to work with python 3 and Boto3. You might need to run

/Applications/Python\ 3.6/Install\ Certificates.command

from your mac command line if you are using python 3.6 and are getting ssl error issues when deleting file.

So it was not an aws or permission problem... but a python3 bytes / string issue. Also there was a type coercion issue with this line which always returned false:

    return bucket == settings.AWS_EXPECTED_BUCKET and parsed_max_size == settings.AWS_MAX_SIZE

which prevented the display of any meaningful error message.

from django.conf import settings
from django.http import HttpResponse
from django.shortcuts import render
from django.views.decorators.csrf import csrf_exempt

import base64, hmac, hashlib, json

import boto3

# Enforce session to inject credentials
session = boto3.Session(
        aws_access_key_id = settings.AWS_SERVER_PUBLIC_KEY,
        aws_secret_access_key = settings.AWS_SERVER_SECRET_KEY,
)
S3 = session.resource( 's3' )


def video_create_form( request ):
    return render( request, 'video_create_form_view.html' )


@csrf_exempt
def success_redirect_endpoint( request ):
    """ This is where the upload will send a POST request after the
    file has been stored in S3.
    """
    return make_response( 200 )


@csrf_exempt
def handle_s3( request ):
    """ View which handles all POST and DELETE requests sent by Fine Uploader
    S3. You will need to adjust these paths/conditions based on your setup.
    """
    if request.method == "POST":
        return handle_POST( request )
    elif request.method == "DELETE":
        return handle_DELETE( request )
    else:
        return HttpResponse( status = 405 )


def handle_POST( request ):
    """ Handle S3 uploader POST requests here. For files <=5MiB this is a simple
    request to sign the policy document. For files >5MiB this is a request
    to sign the headers to start a multipart encoded request.
    """
    class MyEncoder( json.JSONEncoder ):
        """Converts a dict of bytes to Json"""
        def default( self, obj ):
            if isinstance( obj, (bytes, bytearray) ):
                return obj.decode( "ASCII" )  # <- or any other encoding of your choice
            # Let the base class default method raise the TypeError
            return json.JSONEncoder.default( self, obj )

    if request.POST.get( 'success', None ):
        return make_response( 200 )
    else:
        request_payload = json.loads( request.body )
        headers = request_payload.get( 'headers', None )
        if headers:
            # The presence of the 'headers' property in the request payload
            # means this is a request to sign a REST/multipart request
            # and NOT a policy document
            response_data = sign_headers( headers )
        else:
            if not is_valid_policy( request_payload ):
                return make_response( 400, { 'invalid': True } )
            response_data = sign_policy_document( request_payload )
        response_payload = json.dumps( response_data, cls = MyEncoder )
        return make_response( 200, response_payload )


def handle_DELETE( request ):
    """ Handle file deletion requests. For this, we use the Amazon Python SDK, boto.
    """
    if boto3:
        bucket_name = request.GET.get( 'bucket' )
        key_name = request.GET.get( 'key' )
        S3.Object( bucket_name, key_name ).delete()

        return make_response( 200 )
    else:
        return make_response( 500 )


def make_response( status = 200, content = None ):
    """ Construct an HTTP response. Fine Uploader expects 'application/json'.
    """
    response = HttpResponse()
    response.status_code = status
    response[ 'Content-Type' ] = "application/json"
    response.content = content
    return response


def is_valid_policy( policy_document ):
    """ Verify the policy document has not been tampered with client-side
    before sending it off.
    """
    # bucket = settings.AWS_EXPECTED_BUCKET
    # parsed_max_size = settings.AWS_MAX_SIZE
    bucket = ''
    parsed_max_size = 0

    for condition in policy_document[ 'conditions' ]:
        if isinstance( condition, list ) and condition[ 0 ] == 'content-length-range':
            parsed_max_size = condition[ 2 ]
        else:
            if condition.get( 'bucket', None ):
                bucket = condition[ 'bucket' ]

    return bucket == settings.AWS_EXPECTED_BUCKET and int(
            parsed_max_size ) == settings.AWS_MAX_SIZE


def sign_policy_document( policy_document ):
    """ Sign and return the policy doucument for a simple upload.
    http://aws.amazon.com/articles/1434/#signyours3postform
    """
    policy_document_string = str.encode( str( policy_document ) )
    policy = base64.b64encode( policy_document_string )
    aws_secret_key = settings.AWS_CLIENT_SECRET_KEY
    secret_key = str.encode( aws_secret_key )

    signature = base64.b64encode(
            hmac.new( secret_key, policy, hashlib.sha1 ).digest() )
    return {
        'policy'   : policy,
        'signature': signature
    }


def sign_headers( headers ):
    """ Sign and return the headers for a chunked upload. """
    headers_bytes = bytearray( headers, 'utf-8' )  # hmac doesn't want unicode
    aws_client_secret = str.encode( settings.AWS_CLIENT_SECRET_KEY )
    return {
        'signature': base64.b64encode(
                hmac.new( aws_client_secret, headers_bytes, hashlib.sha1 ).digest() )
    }

Signature error message when uploading file with fineuploader and django into S3

Answers (1)

Related Questions