CODEWITHSUNDEEP
pythonlistamazon-web-servicesamazon-s3boto3
MouIdri
MouIdri

Reputation: 1390

list parts in a multipart upload using boto3 with python

I am using multipart upload for a specific large object/file. This works fine using this thread ( https://github.com/boto/boto3/issues/50 ) . But when I am trying to list the parts using the list_parts command in boto ( to verify some points) it gives me an error :

here is the code : 

import os
import json
import sys
import boto3
from boto3 import client
from botocore.utils import fix_s3_host

def listbucketandobjects () :
    with open("credentials.json", 'r') as f:
        data = json.loads(f.read())
        bucket_target = data["aws"]["targetBucket"]
        s3ressource = client(
            service_name='s3', 
            endpoint_url= data["aws"]["hostEndPoint"],
            aws_access_key_id= data["aws"]["idKey"],
            aws_secret_access_key=data["aws"]["secretKey"],
            use_ssl=True,
            )           
        key = 'mp-test.txt'
        # Initiate the multipart upload and send the part(s)
        mpu = s3ressource.create_multipart_upload(Bucket=bucket_target, Key=key)        
        part1 = s3ressource.upload_part(Bucket=bucket_target, Key=key, PartNumber=1,
                                UploadId=mpu['UploadId'], Body='Hello, world!')
        # Next, we need to gather information about each part to complete
        # the upload. Needed are the part number and ETag.

        part_info = {
            'Parts': [
                {
                    'PartNumber': 1,
                    'ETag': part1['ETag']
                }
            ]
        }
        # Now the upload works!
        s3ressource.complete_multipart_upload(Bucket=bucket_target, Key=key, UploadId=mpu['UploadId'],
                                        MultipartUpload=part_info)

        for item in mpu['UploadId']:
            print (item)        

        for item in s3ressource.list_parts(Bucket=bucket_target,Key=key, UploadId=mpu['UploadId']):
            print(item)

listpartsobjects ()

here is the error : 

for item in s3ressource.list_parts(Bucket=bucket_target,Key=key, UploadId=mpu['UploadId']):
File "/usr/local/lib/python2.7/dist-packages/botocore/client.py", line 253, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/usr/local/lib/python2.7/dist-packages/botocore/client.py", line 543, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.errorfactory.NoSuchKey: An error occurred (NoSuchKey) when calling the ListParts operation: The specified key does not exist.

But when checking the aws page here ( http://boto3.readthedocs.io/en/latest/reference/services/s3.html?highlight=list%20object ) I think I am missing something. But I do not see what..

Upvotes: 0

Views: 3228

Answers (1)

MouIdri
MouIdri

Reputation: 1390

Thanks to John Rotenstein, it worked, below is the ugly ( I know but this is before cleaning up ) piece of code that does creat the multipart then list parts then complete the upload. 

def listbucketandobjects () :
    with open("credentials.json", 'r') as f:
        data = json.loads(f.read())
        bucket_target = data["aws"]["targetBucket"]
        s3ressource = client(
            service_name='s3', 
            endpoint_url= data["aws"]["hostEndPoint"],
            aws_access_key_id= data["aws"]["idKey"],
            aws_secret_access_key=data["aws"]["secretKey"],
            use_ssl=True,
            )

        key = 'mp-test.txt'
        mpu = s3ressource.create_multipart_upload(Bucket=bucket_target, Key=key)
        part1 = s3ressource.upload_part(Bucket=bucket_target, Key=key, PartNumber=1,
                                UploadId=mpu['UploadId'], Body='Hello, world!')
        # Next, we need to gather information about each part to complete
        # the upload. Needed are the part number and ETag.
        part_info = {
            'Parts': [
                {
                    'PartNumber': 1,
                    'ETag': part1['ETag']
                }
            ]
        }
        IDofUploadedMPU=mpu['UploadId']
        print ('**********************PRINT THE  ULPOAD ID **********************')
        print IDofUploadedMPU
        print ('**********************PRINT THE COMPLETE LIST PARTS  **********************')
        jacko=s3ressource.list_parts(Bucket=bucket_target,Key=key,UploadId=IDofUploadedMPU)
        print (jacko)
        print ('**********************PRINT THE RECURSIVE COMPLETE LIST PARTS  **********************')
        for jack in s3ressource.list_parts(Bucket=bucket_target,Key=key, UploadId=IDofUploadedMPU)["Parts"]:
            print(jack)
        print ('********************** NOW UPLOADING **********************')
        s3ressource.complete_multipart_upload(Bucket=bucket_target, Key=key, UploadId=mpu['UploadId'],
                                        MultipartUpload=part_info)

listbucketandobjects ()

Upvotes: 1

Related Questions