CODEWITHSUNDEEP
pythonseleniumselenium-webdriverweb-scrapingbeautifulsoup
Rman n
Rman n

Reputation: 45

python selenium driver not scrolling to collect all data points

I am attempting to get some data from the EPA website, unfortunately i am not able to capture all of the data points, I theorize this is due to a combination of scrolling and waiting for the tag to become visible. however I have been working on this since yesterday with no luck.

from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from selenium.webdriver.common.keys import Keys
import numpy as np

options = webdriver.ChromeOptions()
path = '/Users/<user>/Applications/chromedriver'
options.set_headless(True) 

driver = webdriver.Chrome(chrome_options= options, executable_path=path)
url = 'https://edap.epa.gov/public/single/?appid=73b2b6a5-70c6-4820-b3fa-186ac094f10d&obj=b5bf280c-3488-4e46-84f6-58e2a0c34108&opt=noanimate%2Cnoselections&select=clearall'

driver.set_window_size(1920, 1080)
driver.get(url)



SCROLL_PAUSE_TIME = 0.5

# Get scroll height
last_height = driver.execute_script("return document.body.scrollHeight")

rin_data = []

while True:
    # Scroll down to bottom
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.CLASS_NAME,"qv-st-value-overflow")))
    soup = BeautifulSoup(driver.page_source, "html.parser")
    tableURL = soup.select('.qv-st-value-overflow')
    for rin_val in tableURL:
        rin_data.append(rin_val.get_text())

    # Wait to load page
    time.sleep(SCROLL_PAUSE_TIME)

    # Calculate new scroll height and compare with last scroll height
    new_height = driver.execute_script("return document.body.scrollHeight")
    if new_height == last_height:
        break
    last_height = new_height

Upvotes: 0

Views: 985

Answers (1)

ewwink
ewwink

Reputation: 19154

It use Websocket not Ajax to get the data and you need to scroll the table[ng-style="tableStyles.content"] not body but it need custom scroll or using mouse wheel scroll. The function is taken from here

SCROLL_PAUSE_TIME = 2

driver.get(url)

# add mouse wheel function to the page
driver.execute_script('''
window.scrollTable = function() {
  var element = document.querySelector('table[ng-style="tableStyles.content"]')
  var box = element.getBoundingClientRect();
  var deltaY = box.height;
  var clientX = box.left + (box.width / 2);
  var clientY = box.top + (box.height / 2);
  var target = element.ownerDocument.elementFromPoint(clientX, clientY);

  for (var e = target; e; e = e.parentElement) {
    if (e === element) {
      target.dispatchEvent(new WheelEvent('wheel', {view: window, bubbles: true, cancelable: true, clientX: clientX, clientY: clientY, deltaY: deltaY}));
    }
  }
}
''')

rin_data = []

while True:
    WebDriverWait(driver, 30).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, 'tr[class^="qv-st-data-row"]'))
    )
    last_position = driver.find_element_by_css_selector(".scrollbar-thumb").get_attribute('style')
    rows = driver.find_elements_by_css_selector('tr[class^="qv-st-data-row"]')
    for row in rows:
        rin_data.append(row.text)

    # Scroll down the table
    driver.execute_script('scrollTable()')
    # Wait to load content from Websocket, maybe need to increase
    time.sleep(SCROLL_PAUSE_TIME)

    # Calculate new scroll position and compare with last scroll position
    new_position = driver.find_element_by_css_selector(".scrollbar-thumb").get_attribute('style')
    if new_position == last_position:
        break

Note, in this case you don't need to use BeautifulSoup

Upvotes: 1

Related Questions