CODEWITHSUNDEEP
pythonseleniumselenium-webdriverbeautifulsoup
EveryDayDev
EveryDayDev

Reputation: 43

Web Scraping with Selenium and BeautifulSoup returns Empty list

I am trying to search through websites to determine with certain items are in stock. I want this to search the site, find out if the item is "In stock" or not and notify the user of the found information. I can get most of the code to run but this continually returns empty lists. (I tried in BeautifulSoup4 as well as Selenium with the same outcome. Both versions below).What am I missing? Any help would be greatly appreciated!

#selenium version
import os
import string
import subprocess
import traceback
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait

def webscan():
    driver = webdriver.Chrome('/usr/lib/python3/dist-packages/selenium/webdriver/chrome/chromedriver')
    #driver.maximize_window()
    driver.get("https://www.amazon.com/GIGABYTE-GeForce-Windforce-Graphics-GV-N207SWF3OC-8GD/dp/B07WN6RVHH/ref=sr_1_11?crid=OOF3E7T9W54&dchild=1&keywords=graphics+card&qid=1630118746&sprefix=graphi%2Caps%2C190&sr=8-11")

    #wait for the page to load fully

    time.sleep(5)

    status = driver.find_element_by_class('a-size-medium a-color-state')
    ps5_avail = status


    print(ps5_avail)
    if ps5_avail == "Currently unavailable":
        print("Sorry. Still not in stock...")
    elif ps5_avail == "Temporarily Out of Stock":
        print("Sorry. Still not in stock...")
    elif ps5_avail == "In stock soon.":
        print("sorr. Still not in stock...")
    else:
        print("Go NOW!!!!!")


    driver.close()


user_input = (input("Want a PS5?"))

if user_input == 'yes':
    webscan()
    print('scan is done')


####################################################################
#beautifulsoup version

import requests
from bs4 import BeautifulSoup

url = 'https://www.amazon.com/GIGABYTE-GeForce-Windforce-Graphics-GV-N207SWF3OC-8GD/dp/B07WN6RVHH/ref=sr_1_11?crid=OOF3E7T9W54&dchild=1&keywords=graphics+card&qid=1630118746&sprefix=graphi%2Caps%2C190&sr=8-11'
res = requests.get(url)
html_page = res.content

soup = BeautifulSoup(html_page, 'html.parser')

text = soup.find_all(string='In Stock')

if text == True:
    print("Out of Stock")
else:
    print("Get it!")

Upvotes: 1

Views: 151

Answers (1)

cruisepandey
cruisepandey

Reputation: 29362

You should use this css selector instead :

Code :

driver.get("https://www.amazon.com/GIGABYTE-GeForce-Windforce-Graphics-GV-N207SWF3OC-8GD/dp/B07WN6RVHH/ref=sr_1_11?crid=OOF3E7T9W54&dchild=1&keywords=graphics+card&qid=1630118746&sprefix=graphi%2Caps%2C190&sr=8-11")

#wait for the page to load fully

time.sleep(5)

lngth = len(driver.find_elements(By.CSS_SELECTOR, "span.a-size-medium.a-color-state"))
print("value of lngth", lngth)

try:
    if len(driver.find_elements(By.CSS_SELECTOR, "span.a-size-medium.a-color-state")) > 0:
        print("Inside if")
        status = WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div#availability span"))).text
        print(status)
    else:
        print("Looks like element was not found.")
except:
    print("Something went wrong")
    pass

Imports :

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

Upvotes: 1

Related Questions