Extract iFrame content using Selenium and BeautifulSoup

Question

Sorry, this is my first post, so forgive me for all that I don't know yet! Thanks.

I am trying to complete the following form and extract the associating premium. When I run my code I would expect the annual premium of $156 to be extracted, but all I get is "Annual Premium: -"

Here is the code:

from selenium import webdriver
import time
from bs4 import BeautifulSoup
import requests

driver = webdriver.Chrome(r"C:\Users\tomwp\Downloads\chromedriver_win32\chromedriver.exe")
page = driver.get("https://www.earthquakeauthority.com/")

xpath = '//*[@id="form"]/header/div[2]/a'
btn = driver.find_element_by_xpath(xpath)
btn.click()

time.sleep(5)

iframe = driver.find_element_by_xpath("//iframe[@id='premiumCalc-iframe']")
driver.switch_to.frame(iframe)

xpath = '//*[@id="cea-page-1"]/div/div/div[1]/div/button[1]'
btn = driver.find_element_by_xpath(xpath)
btn.click()

xpath = '//*[@id="startdate"]'
incept_date = driver.find_element_by_xpath(xpath)
incept_date.send_keys("03/24/2019")

xpath = '//*[@id="participatingInsurer"]'
insurance_company = driver.find_element_by_xpath(xpath)
insurance_company.send_keys("Other")

xpath = '//*[@id="street"]'
street_address = driver.find_element_by_xpath(xpath)
street_address.send_keys("26 Los Indios")

xpath = '//*[@id="zipcode"]'
zip_code = driver.find_element_by_xpath(xpath)
zip_code.send_keys("92618")

xpath = '//*[@id="form-views"]/div[18]/div/button'
btn = driver.find_element_by_xpath(xpath)
btn.click()

xpath = '//*[@id="yearbuilt"]'
year_built = driver.find_element_by_xpath(xpath)
year_built.send_keys("2011")

xpath = '//*[@id="insuredvalue"]'
year_built = driver.find_element_by_xpath(xpath)
year_built.send_keys("100000")

xpath = '//*[@id="numberOfStories"]'
number_stories = driver.find_element_by_xpath(xpath)
number_stories.send_keys("Greater than one")

xpath = '//*[@id="foundationtype"]'
foundation = driver.find_element_by_xpath(xpath)
foundation.send_keys("slab")

xpath = '//*[@id="form-views"]/div[14]/div/button'
btn = driver.find_element_by_xpath(xpath)
btn.click()

soup = BeautifulSoup(driver.page_source, 'lxml')
premium = soup.find('div', class_='gauge-subtitle ng-binding ng-scope')
print(premium.text)

This is the $156 I would like to extract:

<div ng-if="isQuoting == false" class="gauge-subtitle ng-binding ng-scope">Annual Premium: $156.00</div>

Note that iframe id is as follows (not sure if this is helpful):

<iframe id="premiumCalc-iframe" style="width: 100%; border: none; height: 1397px;" scrolling="no" src="//calc.earthquakeauthority.com/app/index.html" cd_frame_id_="d0b3a5bcdcfe60ced66a29d282ad86c6"></iframe>

enter image description here

Answer 1

I have tried to make it a little more robust by adding in wait conditions. Your final quote page refreshes at the last click so you likely got a stale element exception. If you can find a decent indicator of that refresh completing you should replace my current time.sleep.

Personally, I would use CSS selectors throughout but I am sticking with xpath to align with your code.

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Chrome(r"C:\Users\tomwp\Downloads\chromedriver_win32\chromedriver.exe")
page = driver.get("https://www.earthquakeauthority.com/")

xpath = '//*[@id="form"]/header/div[2]/a'
btn = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, xpath)))
btn.click()

iframe = WebDriverWait(driver,5).until(EC.presence_of_element_located((By.XPATH, "//iframe[@id='premiumCalc-iframe']")))
driver.switch_to.frame(iframe)

xpath = '//*[@id="cea-page-1"]/div/div/div[1]/div/button[1]'
btn = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, xpath)))
btn.click()


xpath = '//*[@id="startdate"]'
incept_date = WebDriverWait(driver,5).until(EC.presence_of_element_located((By.XPATH, xpath)))
incept_date.send_keys("03/24/2019")

xpath = '//*[@id="participatingInsurer"]'
insurance_company = driver.find_element_by_xpath(xpath)
insurance_company.send_keys("Other")

xpath = '//*[@id="street"]'
street_address = driver.find_element_by_xpath(xpath)
street_address.send_keys("26 Los Indios")

xpath = '//*[@id="zipcode"]'
zip_code = driver.find_element_by_xpath(xpath)
zip_code.send_keys("92618")

xpath = '//*[@id="form-views"]/div[18]/div/button'
btn = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, xpath)))
btn.click()


xpath = '//*[@id="yearbuilt"]'
year_built = WebDriverWait(driver,5).until(EC.presence_of_element_located((By.XPATH, xpath)))
year_built.send_keys("2011")

xpath = '//*[@id="insuredvalue"]'
year_built = driver.find_element_by_xpath(xpath)
year_built.send_keys("100000")

xpath = '//*[@id="numberOfStories"]'
number_stories = driver.find_element_by_xpath(xpath)
number_stories.send_keys("Greater than one")

xpath = '//*[@id="foundationtype"]'
foundation = driver.find_element_by_xpath(xpath)
foundation.send_keys("slab")

xpath = '//*[@id="form-views"]/div[14]/div/button'
btn = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, xpath)))
btn.click()

time.sleep(2)

quote = driver.find_element_by_css_selector(".gauge-subtitle").text

print(quote)

Answer 2

if I get you correct, you are able to navigate upto estimation page and able to see estimated annual premium value.

If thats the case then just try this code:

iframe = driver.find_element_by_xpath("//iframe[@id='premiumCalc-iframe']")
yourResult = driver.find_element_by_class_name("gauge-subtitle ng-binding ng-scope").text