How to insert special characters in tkinter

Question

I'm creating a GUI with tkinter, in this GUI there are some entry widgets that the user has to fill in, and the inputs will be stored in a file. The issue that I'm facing is that the user can't insert special characters like emojis in these entry widgets.

I found some ways to display them...:

• converting a special character to the equivalent surrogate pair: this '\U0001f64f'(🙏) to this '\ud83d\ude4f' Python: Find equivalent surrogate pair from non-BMP unicode char

• converting a special character to the equivalen javascript format: this 😊 to this '\uD83D\uDE05' Displaying emojis/symbols in Python using tkinter lib

... but I didn't find anything about my problem.
I thought about solving it by displaying near the entries a table with the emojis and the relative code to insert it in the entry instead of inserting the emoji directly, so something like this...:
1:🙏 2:😊 3:🔥 4:😃
...and have the user insert 1, 2, 3 or 4 instead of the emoji, but I don't think that this is a good way to solve the problem.

From my research, I understood that the problem is in the tkinter module and I was wondering if there was a way to overcome it.

# -*- coding: utf-8 -*-
from tkinter import *

def sumbit():
    print(var.get())

root = Tk()
root.tk.call('encoding', 'system', 'utf-8')

var = StringVar()
entry = Entry(root, textvariable=var)
entry.pack()
button = Button(root, text="sumbit", command=sumbit)
button.pack()

root.mainloop()

This is an example of the problem, I have a window with an Entry widget where the user can insert an emoji, but if the user inserts an emoji, the code raises the following error:

Exception in Tkinter callback
Traceback (most recent call last):
  File "C:\Program Files\Python37-32\lib\tkinter\__init__.py", line 1705, in __call__
    return self.func(*args)
  File "C:/Users/mcara/PycharmProjects/1/python/1.py", line 5, in sumbit
    print(var.get())
  File "C:\Program Files\Python37-32\lib\tkinter\__init__.py", line 484, in get
    value = self._tk.globalgetvar(self._name)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xed in position 0: invalid continuation byte

I'm working with Windows and Pyton 3.7

Answer 1

I have worked around this problem as this for now:

# -*- coding: utf-8 -*-
from tkinter import *
import tkinter.font as tkFont

# Dict with all the emojis
# it is made in this format -->    EMOJI_NAME: EMOJI_SURROGATE_PAIR
emoji_dict = {
    "GRINNING_FACE": '\ud83d\ude00',
    "GRINNING_FACE_WITH_BIG_EYES": '\ud83d\ude03',
    "GRINNING_FACE_WITH_SMILING_EYES": '\ud83d\ude04',
    "BEAMING_FACE_WITH_SMILING_EYES": '\ud83d\ude01',
    "GRINNING_SQUINTING_FACE": '\ud83d\ude06',
    "GRINNING_FACE_WITH_SWEAT": '\ud83d\ude05',
    "LAUGHING_ON_THE_FLOOR": '\ud83e\udd23',
    "TEARS_OF_JOY": '\ud83d\ude02',
    "SMILING_FACE_SLIGHTLY": '\ud83d\ude42',
    "UPSIDE-DOWN_FACE": '\ud83d\ude43',
    "WINKING_FACE": '\ud83d\ude09',
}

emoji_num_name = dict()
emoji_name_num = dict()
counter = 0
for key in emoji_dict:
    emoji_num_name[counter] = key
    emoji_name_num[key] = counter
    counter += 1


def search(text):
    for widget in emoji_frame.winfo_children():
        if isinstance(widget, Button):
            widget.destroy()

    emoji_name_list = list(emoji_dict.keys())
    emoji_name_list.sort()
    if text == "" or text == " ":
        creates_emojis()
    else:
        x = 10
        y = 0
        for emoji_name in emoji_name_list:
            if emoji_name.startswith(text):
                emoji_code = emoji_dict[emoji_name]
                code_ = emoji_name_num[emoji_name]
                emoji_button = Button(emoji_frame, text=emoji_code, borderwidth=0, font=customFont)
                emoji_button.place(x=x, y=y)
                emoji_button.bind("<Button-1>", lambda event, code=code_, var=sumbit_var: insert_emoji(var, ":-" + str(code) + "-:"))

                if x <= 150:
                    x += 30
                else:
                    x = 10
                    y += 30
        emoji_frame.configure(widt=200, height=y+60)


def insert_emoji(var, code):
    var.set(var.get() + code)


def creates_emojis():
    x = 10
    y = 0
    for emoji_name in emoji_dict:
        emoji_code = emoji_dict[emoji_name]
        code_ = emoji_name_num[emoji_name]
        emoji_button = Button(emoji_frame, text=emoji_code, borderwidth=0, font=customFont)
        emoji_button.place(x=x, y=y)
        emoji_button.bind("<Button-1>", lambda event, code=code_, var=sumbit_var: insert_emoji(var, ":-" + str(code) + "-:"))

        if x <= 150:
            x += 30
        else:
            x = 10
            y += 30
    emoji_frame.configure(widt=200, height=y+60)


def sumbit(text):
    text = text.split(":-")
    for index in range(len(text)):
        word = text[index]
        word = word.split("-:")
        for index_ in range(len(word)):
            little_word = word[index_]
            if little_word.isdigit():
                emoji_name = emoji_num_name[int(little_word)]
                emoji = emoji_dict[emoji_name]
                word[index_] = emoji
        text[index] = "".join(word)
    text = "".join(text)
    text = text.encode('utf-16', 'surrogatepass').decode('utf-16')
    print(text)


root = Tk()
root.tk.call('encoding', 'system', 'utf-8')
root.configure(width=500, height=500)
font = "Courier"
customFont = tkFont.Font(family=font, size=14)

emoji_frame = LabelFrame(text="emojis")
emoji_frame.place(x=10, y=60)

search_var = StringVar()
search_entry = Entry(root, textvariable=search_var)
search_entry.place(x=10, y=10)
search_button = Button(root, text="search", command=lambda: search(search_var.get().upper()))
search_button.place(x=10, y=30)
displat_all_button = Button(root, text="display all", command=lambda: creates_emojis())
displat_all_button.place(x=60, y=30)

sumbit_var = StringVar()
sumbit_entry = Entry(root, textvariable=sumbit_var)
sumbit_entry.place(x=200, y=10)
sumbit_button = Button(root, text="sumbit", command=lambda: sumbit(sumbit_var.get()))
sumbit_button.place(x=200, y=30)
creates_emojis()

root.mainloop()

This is a runnable example of what i made, I've created a kind of table where you can insert as many emojis as you want (by editing the emoji_dict and inserting the emoji that you want) and return an output in utf-8.

For find the emoji surrogate pair i've used the code

import re

_nonbmp = re.compile(r'[\U00010000-\U0010FFFF]')

def _surrogatepair(match):
    char = match.group()
    assert ord(char) > 0xffff
    encoded = char.encode('utf-16-le')
    return (
        chr(int.from_bytes(encoded[:2], 'little')) +
        chr(int.from_bytes(encoded[2:], 'little')))

def with_surrogates(text):
    return _nonbmp.sub(_surrogatepair, text)


emoji_dict = {
    "Grinning_Face": u'\ud83d\ude00',
    "Grinning_Face_With_Big_Eyes": u'\ud83d\ude03',
    "Grinning_Face_With_Smiling_Eyes": u'\ud83d\ude04',
    "Beaming_Face_With_Smiling_Eyes": u'\ud83d\ude01',
    "Grinning_Squinting_Face": u'\ud83d\ude06',
    "Grinning_Face_With_Sweat": u'\ud83d\ude05',
    "Laughing_on_the_Floor": u'\ud83e\udd23',
    "Tears_of_Joy": u'\ud83d\ude02',
    "Smiling_Face_Slightly": u'\ud83d\ude42',
    "Upside-Down_Face": u'\ud83d\ude43',
    "Winking_Face": u'\ud83d\ude09',
}

emoji_list =[ "😀", "😃", "😄", "😁", "😆", "😅", "🤣", "😂", "🙂",  "🙃", "😉", ]


for emoji in emoji_list:
    print(repr(_nonbmp.sub(_surrogatepair, emoji)))

you can found it at this question Python: Find equivalent surrogate pair from non-BMP unicode char