Explicit wait returns TimeOutException in python selenium webdriver

Question

I made a code to track which FPL players are on the top 50 fpl managers. The code is working fine, but i wanted to replace time.sleep() with explicit wait to save some time. However, python just returns TimeOutException, and i have no idea how to fix it. I have tried using implicit wait, but that did not work either. The code simply got stuck after trying to find the first element. I am a python newbie, the code may not look very clean. I would appreciate if you could post your code suggestion with the answer.

Error:

Traceback (most recent call last):
  File "C:\Users\William\PycharmProjects\pythonProject\venv\Lib\site-packages\sympy\this.py", line 26, in <module>
    def_list.append(WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH, f'/html/body/main/div/div[2]/div[2]/div[1]/div[4]/div/div/div/div/div[1]/div[2]/div[{k}]/div/div/button/div/div[1]'))).text)
  File "C:\Users\William\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\support\wait.py", line 80, in until
    raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message: 

Working code with time.sleep():

from selenium import webdriver
from selenium.webdriver.common.by import By
from time import sleep
from selenium.common.exceptions import NoSuchElementException

driver = webdriver.Chrome(r'C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe')
wait = 1.5

gk_list, def_list, mid_list, attack_list, subst_list = [],[],[],[],[]

driver.get('https://fantasy.premierleague.com/leagues/314/standings/c')

sleep(wait)

driver.find_element(By.XPATH, '/html/body/div[2]/div/div/div[1]/div[5]/button[1]').click() #Accepts cookies

for i in range(1,51): #Loop with common XPATH numbers for top 50 FPL players
    navn = driver.find_element(By.XPATH, f'/html/body/main/div/div[2]/div[2]/div[1]/div/table/tbody/tr[{i}]/td[2]/a/strong')
    sleep(wait)
    navn.click() #Clicks on the user
    sleep(wait)
    
    #Trying to find the goalkeeper
    gk_list.append(driver.find_element(By.XPATH, '/html/body/main/div/div[2]/div[2]/div[1]/div[4]/div/div/div/div/div[1]/div[1]/div[3]/div/div/button/div/div[1]').text)
    for k in range(1,6):
        try: #Trying to find the defenders
            def_list.append(driver.find_element(By.XPATH, f'/html/body/main/div/div[2]/div[2]/div[1]/div[4]/div/div/div/div/div[1]/div[2]/div[{k}]/div/div/button/div/div[1]').text)
        except NoSuchElementException: #Skips elements that does not exist
            pass

        try: #Trying to find the midfielders
            mid_list.append(driver.find_element(By.XPATH, f'/html/body/main/div/div[2]/div[2]/div[1]/div[4]/div/div/div/div/div[1]/div[3]/div[{k}]/div/div/button/div/div[1]').text)
        except NoSuchElementException:
            pass

        try: #Trying to find the attackers
            attack_list.append(driver.find_element(By.XPATH, f'/html/body/main/div/div[2]/div[2]/div[1]/div[4]/div/div/div/div/div[1]/div[4]/div[{k}]/div/div/button/div/div[1]').text)
        except NoSuchElementException:
            pass

        try: #Trying to find the subs
            subst_list.append(driver.find_element(By.XPATH, f'/html/body/main/div/div[2]/div[2]/div[1]/div[4]/div/div/div/div/div[2]/div/div[{k}]/div/div/button/div/div[1]').text)
        except NoSuchElementException:
            pass

    sleep(wait)
    driver.back() #Returns to main page


driver.quit()


gk_dict = dict() #Converts list to dictionary, key is player, value is player occurrence
for i in gk_list:
    gk_dict[i] = gk_dict.get(i,0) + 1

def_dict = dict() 
for i in def_list:
    def_dict[i] = def_dict.get(i,0) + 1

mid_dict = dict() 
for i in mid_list:
    mid_dict[i] = mid_dict.get(i,0) + 1

attack_dict = dict() 
for i in attack_list:
    attack_dict[i] = attack_dict.get(i,0) + 1

subst_dict = dict() 
for i in subst_list:
    subst_dict[i] = subst_dict.get(i,0) + 1

gk_sorted = {k: v for k, v in sorted(gk_dict.items(), key = lambda v: v[1], reverse=True)} #Sorts the dictionaries from high values to low
def_sorted = {k: v for k, v in sorted(def_dict.items(), key = lambda v: v[1], reverse=True)}
mid_sorted = {k: v for k, v in sorted(mid_dict.items(), key = lambda v: v[1], reverse=True)}
attack_sorted = {k: v for k, v in sorted(attack_dict.items(), key = lambda v: v[1], reverse=True)}
subst_sorted = {k: v for k, v in sorted(subst_dict.items(), key = lambda v: v[1], reverse=True)}

print("\nKeepers:\n", gk_sorted,"\n")
print("\nDefenders:\n", def_sorted,"\n")
print("\nMidfielders:\n", mid_sorted,"\n")
print("\nAttackers:\n", attack_sorted,"\n")
print("\nSubs:\n", subst_sorted,"\n")

Code that is not working with excplicit wait:

from selenium import webdriver
from selenium.webdriver.common.by import By
from time import sleep
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC


driver = webdriver.Chrome(r'C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe')

gk_list, def_list, mid_list, attack_list, subst_list = [],[],[],[],[]

driver.get('https://fantasy.premierleague.com/leagues/314/standings/c')


WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[2]/div/div/div[1]/div[5]/button[1]'))).click() #Accepts cookies

for i in range(1,51): #Loop with common XPATH numbers for top 50 FPL players
    navn = WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH, f'/html/body/main/div/div[2]/div[2]/div[1]/div/table/tbody/tr[{i}]/td[2]/a/strong')))
    navn.click() #Clicks on the user
    
    #Trying to find to goalkeeper
    gk_list.append(WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH, '/html/body/main/div/div[2]/div[2]/div[1]/div[4]/div/div/div/div/div[1]/div[1]/div[3]/div/div/button/div/div[1]'))).text)
    for k in range(1,6):
        try: #Trying to find the defenders
            def_list.append(WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH, f'/html/body/main/div/div[2]/div[2]/div[1]/div[4]/div/div/div/div/div[1]/div[2]/div[{k}]/div/div/button/div/div[1]'))).text)
        except NoSuchElementException:
            pass

        try: #Trying to find the midfielders
            mid_list.append(WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH, f'/html/body/main/div/div[2]/div[2]/div[1]/div[4]/div/div/div/div/div[1]/div[3]/div[{k}]/div/div/button/div/div[1]'))).text)
        except NoSuchElementException:
            pass

        try: #Trying to find the attackers
            attack_list.append(WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH, f'/html/body/main/div/div[2]/div[2]/div[1]/div[4]/div/div/div/div/div[1]/div[4]/div[{k}]/div/div/button/div/div[1]'))).text)
        except NoSuchElementException:
            pass

        try: #Trying to find the subs
            subst_list.append(WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH, f'/html/body/main/div/div[2]/div[2]/div[1]/div[4]/div/div/div/div/div[2]/div/div[{k}]/div/div/button/div/div[1]'))).text)
        except NoSuchElementException:
            pass

    driver.back() #Returns to main page


driver.quit()


gk_dict = dict() #Converts list to dictionary, key is player, value is player occurrence
for i in gk_list:
    gk_dict[i] = gk_dict.get(i,0) + 1

def_dict = dict() 
for i in def_list:
    def_dict[i] = def_dict.get(i,0) + 1

mid_dict = dict() 
for i in mid_list:
    mid_dict[i] = mid_dict.get(i,0) + 1

attack_dict = dict() 
for i in attack_list:
    attack_dict[i] = attack_dict.get(i,0) + 1

subst_dict = dict() 
for i in subst_list:
    subst_dict[i] = subst_dict.get(i,0) + 1

gk_sorted = {k: v for k, v in sorted(gk_dict.items(), key = lambda v: v[1], reverse=True)} #Sorts the dictionaries from high values to low
def_sorted = {k: v for k, v in sorted(def_dict.items(), key = lambda v: v[1], reverse=True)}
mid_sorted = {k: v for k, v in sorted(mid_dict.items(), key = lambda v: v[1], reverse=True)}
attack_sorted = {k: v for k, v in sorted(attack_dict.items(), key = lambda v: v[1], reverse=True)}
subst_sorted = {k: v for k, v in sorted(subst_dict.items(), key = lambda v: v[1], reverse=True)}

print("\nKeepers:\n", gk_sorted,"\n")
print("\nDefenders:\n", def_sorted,"\n")
print("\nMidfielders:\n", mid_sorted,"\n")
print("\nAttackers:\n", attack_sorted,"\n")
print("\nSubs:\n", subst_sorted,"\n")

Answer 1

I understand that your code might be working, but having such absolute xpath locators would make it flaky for sure. Try to keep your locators as relative as possible. I have refactored your code to an extent with relative locators as much as possible, the approach, you can take and improve if you would like to. Mind you that it is not extensive code, which implies that you cannot take it blindly and and run it. It is prone to errors (especially, you may get stale element issues when the driver navigates to back page. I am not sure if you encountered it or not in your version though). Also, I have used list view instead of field view, which makes it easier to traverse through the the table and find elements instead of finding things on the pitch/field.

gk_list, def_list, mid_list, attack_list, subst_list = [], [], [], [], []

driver.get('https://fantasy.premierleague.com/leagues/314/standings/c')

WebDriverWait(driver, 10).until(EC.element_to_be_clickable(
    (By.XPATH, '/html/body/div[2]/div/div/div[1]/div[5]/button[1]'))).click()  # Accepts cookies

# for i in range(1, 51):  # Loop with common XPATH numbers for top 50 FPL players
WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.TAG_NAME, 'table')))
top_50 = driver.find_elements(By.XPATH, "//table//tbody//tr//td//following-sibling::td/a")
for each_one in top_50:
    print(each_one.text)
    each_one.click()
    WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//a[text()='List View']"))).click()
    players = driver.find_elements(By.XPATH, "//th[text()='Starters']//ancestor::table//tr[contains(@class, 'ElementTable__')]")
    for player in players:
        player.click()

        # Trying to find to goalkeeper
        try:
            gkp = driver.find_elements(By.XPATH, "//span[text()='GKP']//parent::div//preceding-sibling::div")
            for keeper in gkp:
                gk_list.append(keeper.text)
        except NoSuchElementException:
            pass

        try:
            defenders = driver.find_elements(By.XPATH, "//span[text()='DEF']//parent::div//preceding-sibling::div")
            for defender in defenders:
                def_list.append(defender.text)
        except NoSuchElementException:
            pass
        try:
            attackers = driver.find_elements(By.XPATH, "//span[text()='MID']//parent::div//preceding-sibling::div")
            for attacker in attackers:
                attack_list.append(attacker.text)

        except NoSuchElementException:
            pass

    try:
        subs = driver.find_elements(By.XPATH, "//th[text()='Substitutes']//ancestor::table//tr[contains(@class, 'ElementTable__')]")
        for each_sub in subs:
            subst_list.append(each_sub.text)
    except NoSuchElementException:
        pass

    driver.back()  # Returns to main page
    

driver.quit()

gk_dict = dict()  # Converts list to dictionary, key is player, value is player occurrence
for i in gk_list:
    gk_dict[i] = gk_dict.get(i, 0) + 1

def_dict = dict()
for i in def_list:
    def_dict[i] = def_dict.get(i, 0) + 1

mid_dict = dict()
for i in mid_list:
    mid_dict[i] = mid_dict.get(i, 0) + 1

attack_dict = dict()
for i in attack_list:
    attack_dict[i] = attack_dict.get(i, 0) + 1

subst_dict = dict()
for i in subst_list:
    subst_dict[i] = subst_dict.get(i, 0) + 1

gk_sorted = {k: v for k, v in sorted(gk_dict.items(), key=lambda v: v[1],
                                     reverse=True)}  # Sorts the dictionaries from high values to low
def_sorted = {k: v for k, v in sorted(def_dict.items(), key=lambda v: v[1], reverse=True)}
mid_sorted = {k: v for k, v in sorted(mid_dict.items(), key=lambda v: v[1], reverse=True)}
attack_sorted = {k: v for k, v in sorted(attack_dict.items(), key=lambda v: v[1], reverse=True)}
subst_sorted = {k: v for k, v in sorted(subst_dict.items(), key=lambda v: v[1], reverse=True)}

print("\nKeepers:\n", gk_sorted, "\n")
print("\nDefenders:\n", def_sorted, "\n")
print("\nMidfielders:\n", mid_sorted, "\n")
print("\nAttackers:\n", attack_sorted, "\n")
print("\nSubs:\n", subst_sorted, "\n")

As I have said, you may use this as a reference only and improve upon it.