Webdriver do not get attribute of ® Selenium driver

Question

Hi I would like to get to ® symbol to be recognized in python in the file I've shown you bellow

.get_attribute("href")

but I've tried many ways to decode and encode it but it doesn't seem to work. I suspect that it is made by UTF-8 encoding but it cannot be decoded. The thing is the code seems all fine and I've checked through everything the line of the code.

from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
import time
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
import csv

import time
url = 'https://shopee.com.my/search?keyword=mattress'

driver = webdriver.Chrome(executable_path=r'E:/users/Francabicon/Desktop/Bots/others/chromedriver.exe')
driver.get(url)
time.sleep(0.8)

# select language
driver.find_element_by_xpath('//div[@class="language-selection__list"]/button').click()
time.sleep(3)


# scroll few times to load all items 
def clickpy():
    for x in range(10):
        driver.execute_script("window.scrollBy(0,300)")
        time.sleep(0.1)

    # get all links (without clicking)

    all_items = driver.find_elements_by_xpath('//a[@data-sqe="link"]')

    all_urls = []

    s=["-Dr.Alstone-","-Dr.-Alstone-","-Lutfy-Paris-"]

    for item in all_items:
        # This give you whole url of the anchor tag
        url = item.get_attribute('href')
        if "-Dr.Alstone-" in url:
            continue
        else:
            if "-Dr.-Alstone-" in url:
                continue
            else:
                if "/Dr.Alstone-" in url:
                    continue
                else:
                    if "-Simoni-" in url:
                        continue
                    else:
                        if "-Lütfy-" in url:
                            continue
                        else:
                            # You need to remove the preceding values in order to verify href later for clicking
                            urlfinal=url.split('https://shopee.com.my')[1]
                            all_urls.append(c)
    print(all_urls)

    a= len(all_urls)

    print('len:' + str(a))

    # now use links
    i = 0
    j= a-5


    while i <= 4 :
        #Identify the parent tag by child tag use following Xpath.
        try:
            WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[@class='col-xs-2-4 shopee-search-item-result__item' and @data-sqe='item'][.//a[@data-sqe='link' and @href='" + all_urls[i] +"']]"))).click()
            time.sleep(0.8)
            driver.back()
        except:
            print(all_urls[i] + "doesn't work")
            continue
        i+=1

    while j <= a :
        try:
            WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[@class='col-xs-2-4 shopee-search-item-result__item' and @data-sqe='item'][.//a[@data-sqe='link' and @href='" + all_urls[i] +"']]"))).click()
            time.sleep(0.8)
            driver.back()
        except:
            print(all_urls[i] + "doesn't work")
            continue
        j+=1

clickpy()

Is there anything I could do?

Answer 1

You are parsing an HTML page, so all the characters will be in URL encoded UTF-8 format. You can use urllib.parse.unquote (for Python 3) to decode these URL escaped characters.

https://shopee.com.my/LOVOC-%C2%AE-OORIGINAL-Nordic-Sweden-Single-Mattress-(Comfort-in-a-box-Latex-Pocket-Spring-Sweden-Technology-Mattress)-i.132909000.2627476005

The above is one of the URL that contains ® from your example. %C2%AE is the encoded format for ®.

Code:

import urllib.parse
url = 'https://shopee.com.my/LOVOC-%C2%AE-OORIGINAL-Nordic-Sweden-Single-Mattress-(Comfort-in-a-box-Latex-Pocket-Spring-Sweden-Technology-Mattress)-i.132909000.2627476005'
url_decoded = urllib.parse.unquote(url)
print(url_decoded)

Output:

https://shopee.com.my/LOVOC-®-OORIGINAL-Nordic-Sweden-Single-Mattress-(Comfort-in-a-box-Latex-Pocket-Spring-Sweden-Technology-Mattress)-i.132909000.2627476005

Hope this helps you!