Beautiful soup element not found exception

Question

I am making a Crawler using Python 3.6 and Beautiful Soup .This is my code

When I run it I get element not found exception , why? What I want to do is select uri and click on the name uri so a new page opens

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import re
import pandas as pd
import os
url = "https://www.codechef.com/ratings/all?order=asc&page=3&sortBy=global_rank"


# create a new Firefox session
driver = webdriver.Firefox()
driver.get(url)
soup_level1=BeautifulSoup(driver.page_source, 'lxml')

datalist = [] #empty list
x = 488 
for link in soup_level1.find_all('a', id=re.compile(r"^ember")):

    elemnt2222=driver.find_element_by_xpath("//*[@id='ember"+str(493)+"']/td[2]/div[2]/a")

    python_button = elemnt2222
    python_button.click() #click link

Answer 1

You don't have to click. Just get href from anchor. Navigate to that URL.

from selenium import webdriver
from selenium.webdriver.common.keys import Keys    
from bs4 import BeautifulSoup
import re
import pandas as pd
import os
import time

url = "https://www.codechef.com/ratings/all?order=asc&page=3&sortBy=global_rank"
driver = webdriver.Firefox()
driver.get(url)
#Takes some time to load
time.sleep(5)
soup=BeautifulSoup(driver.page_source, 'lxml')

links = soup.select('div.user-name > a')
for link in links:
  print(link.get('href'))

This will give you this result

/users/sumeet_varma
/users/fjzzq2002
/users/dreamoon4
/users/y0105w49
/users/nblt
/users/dzhulgakov
/users/uwi
/users/Fcdkbear
/users/austin990301
/users/KADR
/users/adkroxx
/users/kostroma
/users/fhlasek
/users/argos
/users/watcher
/users/nafis
/users/scli
/users/mister
/users/iwiwi
/users/aurinegro

After that you can navigate to

https://www.codechef.com/users/username