CODEWITHSUNDEEP
pythonseleniumweb-scrapingbeautifulsoup
Buddhika Chathuranga
Buddhika Chathuranga

Reputation: 1468

Unable to fetch data from eBay in Beautiful Soup

I am trying to scrape data from ebay and when i am trying to find a div it always gives me a None. What is the error with my code.

from bs4 import BeautifulSoup
from selenium import webdriver
from openpyxl import Workbook
wb = Workbook()
ws = wb.active

driver = webdriver.Chrome(executable_path = r'E:\chromedriver\chromedriver.exe')

urlBase = "https://www.ebay.com.au/str/Tyroola?rt=nc&_pgn="

for i in range(2, 3):
    url = urlBase + str(i)
    driver.get(url)
    pagehtml = driver.page_source
    soup = BeautifulSoup(pagehtml, 'lxml')

    tyreList = soup.find_all('li', class_='s-item')
    for listItem in tyreList:
        tyre = listItem.find('a')
        driver.get(tyre['href'])
        ebayPage = driver.page_source
        ebaySoup = BeautifulSoup(ebayPage, 'lxml')
        tableDiv = ebaySoup.find('div', id='viTabs_0_is')
        proDiv = ebaySoup.find('div', class_ = 'product-details')
        print(proDiv)

proDiv is always None.

Upvotes: 1

Views: 658

Answers (1)

Vishal Singh
Vishal Singh

Reputation: 6234

the item specifics present on the eBay website are present in the form of an HTML table. enter image description here A total of 5 rows are present inside which exactly 2 <td> tags are present. <td> tag Defines a cell in a table This is one of the ways you can parse this table in Beautiful Soup.

from bs4 import BeautifulSoup
from selenium import webdriver
from openpyxl import Workbook
wb = Workbook()
ws = wb.active

driver = webdriver.Chrome("/usr/lib/chromium-browser/chromedriver")

urlBase = "https://www.ebay.com.au/str/Tyroola?rt=nc&_pgn="

for i in range(2, 3):
    url = urlBase + str(i)
    driver.get(url)
    pagehtml = driver.page_source
    soup = BeautifulSoup(pagehtml, 'lxml')

    tyreList = soup.find_all('li', class_='s-item')
    for listItem in tyreList:
        tyre = listItem.find('a')
        driver.get(tyre['href'])
        ebayPage = driver.page_source
        ebaySoup = BeautifulSoup(ebayPage, 'lxml')
        item_specifics_table = ebaySoup.find('div', class_='section') #table starts here
        rows = item_specifics_table.findAll('tr')#finding all the content of tr tag and saving it into a list:rows
        print
        for cells in rows:#to parse td tags we need to look into each index of the list:rows
            print "{0} {1}\n{2} {3}".format(cells.contents[1].string.strip(), cells.contents[3].find('span').get_text().strip(),
                                            cells.contents[5].string.strip(), cells.contents[7].find('span').get_text().strip())
        # strip() removes excessive whitespace
        print"********************************************************************************"
        print

Upvotes: 1

Related Questions