write utf-8 content in files / python 3

Question

Its again about utf-8 issues, for the 1001st time. Please don't mark this question as duplicate, 'cause I cannot find answers elsewhere.

Since some months I am working successfully with the following small script (which may improved, I know that), which delivers me a simple database functionality. But I wrote it for very simple data storage, like local config and auth data, let's say not for more sophisticated content as known from cookies. It worked for me, until I tried to store non latin characters for the 1st time.

In the following script I already added the import codecs stuff, including the altered lines f = codecs.open(file, 'w', 'utf-8'). Do not know if this is the right approach.

Can somebody show me the trick ? Let's say "John Doe" is french, "John Doé", how do I store it ?

The class itself (to be imported)

#!/usr/bin/python3
# -*- coding: utf-8 -*-

import os, errno
import json
import codecs

class Ddpos:

    def db(self,table,id,col=''):
        table = '/Users/michag/Documents/ddposdb/'+table

        try:
            os.makedirs(table)
            os.chmod(table, 0o755)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        file = table+'/'+id+'.txt'

        if not os.path.isfile(file):
            f = codecs.open(file, 'w', 'utf-8')
            f.write('{}')
            f.close()

        f = codecs.open(file, 'r', 'utf-8')
        r = json.loads(f.readline().strip())
        f.close()

        if isinstance(col, str) and len(col) > 0:
            if col in r:
                return json.dumps(r[col])
            else:
                return ''

        elif isinstance(col, list) and len(col) > 0:
            res = {}
            for el in range(0,len(col)):
                if col[el] in r:
                    res[col[el]] = r[col[el]]
            return json.dumps(res)

        elif isinstance(col, dict) and len(col) > 0:
            for el in col:
                r[el] = col[el]
            f = codecs.open(file, 'w', 'utf-8')
            f.write(json.dumps(r))
            f.close()
            return json.dumps(r)

        else:
            return json.dumps(r)

ddpos = Ddpos()

The call / usage

#!/usr/bin/python3
# -*- coding: utf-8 -*-

from ddpos import *

# set values and return all values as dict
print ('1.: '+ddpos.db('cfg','local',{'admin':'John Doé','email':'johndoe@email.com'}))

# return all values as dict
print ('2.: '+ddpos.db('cfg','local'))

# return one value as string
print ('3.: '+ddpos.db('cfg','local','email'))

# return two or more values as dict
print ('4.: '+ddpos.db('cfg','local',['admin','email']))

It prints and stores this in case of "John Doe"

1.: {"admin": "John Doe", "email": "johndoe@email.com"}
2.: {"admin": "John Doe", "email": "johndoe@email.com"}
3.: "johndoe@email.com"
4.: {"admin": "John Doe", "email": "johndoe@email.com"}

and this in case of french guy "John Doé"

1.: {"email": "johndoe@email.com", "admin": "John Do\u00e9"}
2.: {"email": "johndoe@email.com", "admin": "John Do\u00e9"}
3.: "johndoe@email.com"
4.: {"email": "johndoe@email.com", "admin": "John Do\u00e9"}

For me it is more important to learn and to understand, how it works and why or why not, but to know that there already classes which would do the job for me. Thanks for your support.

Answer 1

UPDATE

I made some improvements. Now the stored dict is human readable (for those unbelievers, like me) and sorted case insensitive. This sorting process for sure consumes a bit of performance, but hey, I will use the read-only process some houndred times more than the write process.

Now the stored dict looks like this:

{
        "auth_descr": "dev unit, office2",
        "auth_email": "me@myemail.com",
        "auth_key": "550e3 **shortened sha256** d73b1",
        "auth_unit_id": "2.3.1",
        "vier": "44é", # utf-8 example
        "Vjier": "vier44" # uppercase example
}

I don't know where I did a mistake in earlier version(s), but if you take a look on the utf-8 example. "é" now is stored as an "é" and not as "\u00e9".

The class now looks like this (with minor changes)

#!/usr/bin/python3
# -*- coding: utf-8 -*-

import os, errno
import json

class Ddpos:

        def db(self,table,id,col=''):
                table = '/ddpos/db/'+table

                try:
                        os.makedirs(table)
                        os.chmod(table, 0o755)
                except OSError as e:
                        if e.errno != errno.EEXIST:
                                raise

                file = table+'/'+id+'.txt'

                if not os.path.isfile(file):
                        f = open(file, 'w', encoding='utf-8')
                        f.write('{}')
                        f.close()

                f = open(file, 'r', encoding='utf-8')
                r = json.loads(f.read().strip())
                f.close()

                if isinstance(col, str) and len(col) > 0:
                        if col in r:
                                return r[col]
                        else:
                                return ''

                elif isinstance(col, list) and len(col) > 0:
                        res = {}
                        for el in range(0,len(col)):
                                if col[el] in r:
                                        res[col[el]] = r[col[el]]
                        return res

                elif isinstance(col, dict) and len(col) > 0:
                        for el in col:
                                r[el] = col[el]
                        w = '{\n'
                        for key in sorted(r, key=lambda y: y.lower()):
                                w += '\t"%s": "%s",\n' % (key, r[key])
                        w = w[:-2]+'\n'
                        w += '}'
                        f = open(file, 'w', encoding='utf-8')
                        f.write(w)
                        f.close()
                        return r

                else:
                        return r

ddpos = Ddpos()

Answer 2

After moderator deceze I answer my own question, with credentials to user mata and python3 itself.

Here's the "new" script. The poor french guy now is renamed to "J€hn Doéß" and he's still alive.

#!/usr/bin/python3
# -*- coding: utf-8 -*-

import os, errno
import json

class Ddpos:

    def db(self,table,id,col=''):
        table = '/Users/michag/Documents/ddposdb/'+table

        try:
            os.makedirs(table)
            os.chmod(table, 0o755)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        file = table+'/'+id+'.txt'

        if not os.path.isfile(file):
            f = open(file, 'w')
            f.write('{}')
            f.close()

        f = open(file, 'r')
        r = json.loads(f.readline().strip())
        f.close()

        if isinstance(col, str) and len(col) > 0:
            if col in r:
                return r[col]
            else:
                return ''

        elif isinstance(col, list) and len(col) > 0:
            res = {}
            for el in range(0,len(col)):
                if col[el] in r:
                    res[col[el]] = r[col[el]]
            return res

        elif isinstance(col, dict) and len(col) > 0:
            for el in col:
                r[el] = col[el]
            f = open(file, 'w')
            f.write(json.dumps(r))
            f.close()
            return r

        else:
            return r

ddpos = Ddpos()