CODEWITHSUNDEEP
pythonseleniumweb-scraping
Dragos Ciupe
Dragos Ciupe

Reputation: 109

Selenium WebDriverWait returning exception

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

Path = "C:\Program Files (x86)\chromedriver.exe"
driver = webdriver.Chrome(Path)

driver.get("https://www.emag.ro/")

search_bar = driver.find_element_by_id("searchboxTrigger")
search_bar.send_keys("laptopuri")
search_bar.send_keys(Keys.RETURN)

main = None

try:
    main = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, "main-container"))
    )
    print("Page loaded,main retrived succesfully")

except:
    driver.quit()

items = main.find_element_by_id("card_grid")
products = items.find_elements_by_css_selector("div.card-item.js-product-data")
count = 0

for product in products:
    raw_name = WebDriverWait(product, 10).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, "h2.card-body.product-title-zone"))
    ).text

    raw_price = WebDriverWait(product, 10).until(
        EC.presence_of_element_located((By.CLASS_NAME, "product-new-price"))
    )

    #Parsing the product name

    raw_name = raw_name.replace("Laptop", "")
    raw_name = raw_name.strip()
    if raw_name.startswith("Apple"):
        sEnd = raw_name.find(",")
    else:
        sEnd = raw_name.find("cu") - 1
    product_name = raw_name[:sEnd]


    #Parsing the product price

    raw_price = raw_price.text[:raw_price.text.find(" ")]
    print(raw_price)
    count += 1

print(f"{count} results returned")
driver.quit()

Code works perfectly fine sometimes,but sometimes i get the error: enter image description here

Please note i am new at this,so an explanation would be very appreciated.I just learned how to use selenium and the reason i transitioned from beautifulsoup because of the lack of wait possibility,and now when trying to use that,i get this error SOMETIMES

Upvotes: 0

Views: 1071

Answers (1)

cruisepandey
cruisepandey

Reputation: 29362

See this :

driver = webdriver.Chrome(Path)

and how have you used it here :

try:
    main = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, "main-container"))
    )
    print("Page loaded,main retrived succesfully")

If you pay attention you would see that, you are using WebDriverWait(driver, 10) and passing driver reference.

But here

 raw_name = WebDriverWait(product, 10).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, "h2.card-body.product-title-zone"))
    ).text

you are passing product in WebDriverWait, which is wrong, you should pass driver reference here. like

raw_name = WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, "h2.card-body.product-title-zone"))
        ).text

This should help you past this issue for sure.

also, make changes here

raw_price = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.CLASS_NAME, "product-new-price"))
    )

This is what we have internally :

class WebDriverWait(object):
    def __init__(self, driver, timeout, poll_frequency=POLL_FREQUENCY, ignored_exceptions=None):
        """Constructor, takes a WebDriver instance and timeout in seconds.

Update 1 :

driver = webdriver.Chrome(driver_path)
driver.maximize_window()
driver.implicitly_wait(50)
driver.get("https://www.emag.ro/")
wait = WebDriverWait(driver, 10)

wait.until(EC.element_to_be_clickable((By.XPATH, "//i[contains(@class,'close')]/parent::button[@class='close']"))).click()
ActionChains(driver).move_to_element(wait.until(EC.visibility_of_element_located((By.XPATH, "//button[contains(@class,'js-accept')]")))).click().perform()

search_bar = driver.find_element_by_id("searchboxTrigger")
search_bar.send_keys("laptopuri")
search_bar.send_keys(Keys.RETURN)

main = None

try:
    main = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "main-container")))
    print("Page loaded,main retrived succesfully")

except:
    driver.quit()

items = main.find_element_by_id("card_grid")
products = driver.find_elements_by_css_selector("div.card-item.js-product-data")
count = 0

for product in products:
    raw_name = product.find_element_by_css_selector("h2.card-body.product-title-zone a").text
    print(raw_name)

    raw_price = product.find_element_by_css_selector("p.product-new-price").text
    print(raw_price)

    #Parsing the product name

    # raw_name = raw_name.replace("Laptop", "").strip()
    # if raw_name.startswith("Apple"):
    #     sEnd = raw_name.find(",")
    # else:
    #     sEnd = raw_name.find("cu") - 1
    # product_name = raw_name[:sEnd]


    #Parsing the product price

    # raw_price = raw_price[raw_price.find(" ")]
    # print(raw_price)
    # count += 1

#print(f"{count} results returned")
driver.quit()

Upvotes: 1

Related Questions