Authorization issue on bluemix: Python swift client library accessing object storage

Question

I am writing a code to access object storage using python swift client library. The intention of the program is to read, write and delete CSV files in object storage using swift client API. May i know what is going wrong. I got this code from a website. Is there any other way to access object storage from python?

import swiftclient.client
from keystoneclient import client
print 'i am fine'
conn = swiftclient.Connection(key="password",
authurl='https://identity.open.softlayer.com',
#auth_version='3',
os_options={"project_id": 'project-id',
"user_id": 'user_id',
"region_name": "dallas"})

container_name = 'test'

# File name for testing
file_name = 'result.csv'

# Create a new container
conn.put_container(container_name)
print "nContainer %s created successfully." % container_name

ERROR:swiftclient:Auth GET failed: https://identity.open.softlayer.com 300 Multiple Choices
Traceback (most recent call last):
  File "/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.py", line 1553, in _retry
    self.url, self.token = self.get_auth()
  File "/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.py", line 1507, in get_auth
    timeout=self.timeout)
  File "/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.py", line 593, in get_auth
    timeout=timeout)
  File "/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.py", line 468, in get_auth_1_0
    http_status=resp.status, http_reason=resp.reason)
ClientException: Auth GET failed: https://identity.open.softlayer.com 300 Multiple Choices

ClientException                           Traceback (most recent call last)
<ipython-input-5-3708d247f0cc> in <module>()
     15 
     16 # Create a new container
---> 17 conn.put_container(container_name)
     18 print "nContainer %s created successfully." % container_name
     19 

/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.pyc in put_container(self, container, headers, response_dict)
   1641         """Wrapper for :func:`put_container`"""
   1642         return self._retry(None, put_container, container, headers=headers,
-> 1643                            response_dict=response_dict)
   1644 
   1645     def post_container(self, container, headers, response_dict=None):

/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.pyc in _retry(self, reset_func, func, *args, **kwargs)
   1551             try:
   1552                 if not self.url or not self.token:
-> 1553                     self.url, self.token = self.get_auth()
   1554                     self.http_conn = None
   1555                 if self.service_auth and not self.service_token:

/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.pyc in get_auth(self)
   1505                                         cacert=self.cacert,
   1506                                         insecure=self.insecure,
-> 1507                                         timeout=self.timeout)
   1508         return self.url, self.token
   1509 

/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.pyc in get_auth(auth_url, user, key, **kwargs)
    591                                           cacert=cacert,
    592                                           insecure=insecure,
--> 593                                           timeout=timeout)
    594     elif auth_version in AUTH_VERSIONS_V2 + AUTH_VERSIONS_V3:
    595         # We are handling a special use case here where the user argument

/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.pyc in get_auth_1_0(url, user, key, snet, **kwargs)
    466         raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
    467                               http_host=conn.host, http_path=parsed.path,
--> 468                               http_status=resp.status, http_reason=resp.reason)
    469     if snet:
    470         parsed = list(urlparse(url))

***ClientException: Auth GET failed: https://identity.open.softlayer.com 300 Multiple Choices***

*****With auth_version enabled:*****
ERROR:swiftclient:Authorization Failure. Authorization failed: The resource could not be found. (HTTP 404)
Traceback (most recent call last):
  File "/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.py", line 1553, in _retry
    self.url, self.token = self.get_auth()
  File "/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.py", line 1507, in get_auth
    timeout=self.timeout)
  File "/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.py", line 617, in get_auth
    auth_version=auth_version)
  File "/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.py", line 543, in get_auth_keystone
    raise ClientException('Authorization Failure. %s' % err)
ClientException: Authorization Failure. Authorization failed: The resource could not be found. (HTTP 404)
i am fine
---------------------------------------------------------------------------
ClientException                           Traceback (most recent call last)
<ipython-input-21-d0874a2c5851> in <module>()
     15 
     16 # Create a new container
---> 17 conn.put_container(container_name)
     18 print "nContainer %s created successfully." % container_name
     19 

/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.pyc in put_container(self, container, headers, response_dict)
   1641         """Wrapper for :func:`put_container`"""
   1642         return self._retry(None, put_container, container, headers=headers,
-> 1643                            response_dict=response_dict)
   1644 
   1645     def post_container(self, container, headers, response_dict=None):

/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.pyc in _retry(self, reset_func, func, *args, **kwargs)
   1551             try:
   1552                 if not self.url or not self.token:
-> 1553                     self.url, self.token = self.get_auth()
   1554                     self.http_conn = None
   1555                 if self.service_auth and not self.service_token:

/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.pyc in get_auth(self)
   1505                                         cacert=self.cacert,
   1506                                         insecure=self.insecure,
-> 1507                                         timeout=self.timeout)
   1508         return self.url, self.token
   1509 

/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.pyc in get_auth(auth_url, user, key, **kwargs)
    615                                                insecure=insecure,
    616                                                timeout=timeout,
--> 617                                                auth_version=auth_version)
    618     else:
    619         raise ClientException('Unknown auth_version %s specified.'

/gpfs/fs01/user/s1ab-0eb9620648b3ce-07c8d52a704f/.local/lib/python2.7/site-packages/swiftclient/client.pyc in get_auth_keystone(auth_url, user, key, os_options, **kwargs)
    541         raise ClientException(msg)
    542     except exceptions.AuthorizationFailure as err:
--> 543         raise ClientException('Authorization Failure. %s' % err)
    544     service_type = os_options.get('service_type') or 'object-store'
    545     endpoint_type = os_options.get('endpoint_type') or 'publicURL'

ClientException: Authorization Failure. Authorization failed: The resource could not be found. (HTTP 404) 

Answer 1

You have to specify the auth_version you are using in the authurl.

Add v3 to the authurl and uncomment the auth_version line.

authurl='https://identity.open.softlayer.com/v3',
auth_version='3',

Answer 2

You get multiple choices for the authentication endpoint. You should specify the auth_version and adjust the authurl accordingly. Just append '/v3' to authurl and it should work.

Here is a small example using the Insert to code - credentials function.

# Insert to code - credentials 
credentials = {
  'auth_url':'https://identity.open.softlayer.com',
  'project':'xxx',
  'project_id':'xxx',
  'region':'dallas',
  'user_id':'xxx',
  'domain_id':'xxx',
  'domain_name':'xxx',
  'username':'xxx',
  'password':"""xxx""",
  'filename':'precipitation.csv',
  'container':'notebooks',
  'tenantId':'xxx'
}

# read CSV file from Object Storage
from StringIO import StringIO
import swiftclient.client as swiftclient
import pandas as pd

conn = swiftclient.Connection(
    key=credentials['password'],
    authurl=credentials['auth_url']+"/v3",
    auth_version='3',
    os_options={
        "project_id": credentials['project_id'],
        "user_id": credentials['user_id'],
        "region_name": credentials['region']})

obj = conn.get_object(credentials['container'], credentials['filename'])
data_df = pd.read_csv(StringIO(obj[1]))
data_df.head()

# Create a new container
container_name = "new_container"
conn.put_container(container_name)

# Write CSV file to Object Storage
import csv
conn.put_object(credentials['container'], "precipitation_2.csv", data_df.to_csv(quoting=csv.QUOTE_ALL, index=False), content_type='text')

There is also an alternative to python-swiftclient by using the requests package and build the requests yourself. Have a look at this sample notebook: https://console.ng.bluemix.net/data/notebooks/samples/Precipitation%20Analysis

In section "Access Object Storage", the function get_file_content(credentials) can be used for reading a file. The corresponding POST or DELETE can be implemented in a similar fashion.