WeatherUnderground Web Scraping - choosing different measurements systems

Question

I've been trying to change measurements systems. Default measurement system on the WeatherUnderground is set on imperial system.

from selenium import webdriver
from selenium.webdriver.common.by import By
from datetime import datetime

    def scrape_weather_data(url, time_selector, temperature_selector, 
    humidity_selector, precip_selector, wind_speed_selector, 
    pressure_selector, amount_selector):
    driver = webdriver.Chrome()  

    driver.get(url)  # Enter The website

    driver.find_element(By.CSS_SELECTOR, '#wuSettings > i').click()

    driver.find_element(By.CSS_SELECTOR, '#wuSettings-quick > div > 
    a:nth-child(2)').click()

    # Generate text file name based on day and hour
    current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
    filename = f'dane_pogodowe_{current_time}.txt'

    # Saving data into a .txt file
    with open(filename, 'a') as file:
    time_elements = driver.find_elements(By.CSS_SELECTOR,time_selector)
    temperature_elements = 
    driver.find_elements(By.CSS_SELECTOR,temperature_selector)
    humidity_elements = 
    driver.find_elements(By.CSS_SELECTOR,humidity_selector)
    precip_elements = 
    driver.find_elements(By.CSS_SELECTOR,precip_selector)
    wind_elements = driver.find_elements(By.CSS_SELECTOR, 
    wind_speed_selector)
    pressure_elements = driver.find_elements(By.CSS_SELECTOR, 
    pressure_selector)
    amount_elements = driver.find_elements(By.CSS_SELECTOR, 
    amount_selector)

    for i in range(len(time_elements)):
        time = time_elements[i].text.strip()
        temperature = temperature_elements[i].text.strip()
        humidity = humidity_elements[i].text.strip()
        precip = precip_elements[i].text.strip()
        wind = wind_elements[i].text.strip()
        pressure = pressure_elements[i].text.strip()
        amount = amount_elements[i].text.strip()

        file.write(f"Czas: {time}\n")
        file.write(f"Temperatura: {temperature}\n")
        file.write(f"Wilgotność: {humidity}\n")
        file.write(f"Szansa opadów: {precip}\n")
        file.write(f"Wiatr: {wind}\n")
        file.write(f"Ciśnienie: {pressure}\n")
        file.write(f"Ilość opadów: {amount}\n")
        file.write("\n")
    file.write("------------------------------\n")

    driver.quit()  # Close the Web browser

     print("Dane pogodowe zostały zapisane.")

    # URL
    url = 'https://www.wunderground.com/hourly/pl/kucoby'

    # Selectors CSS
    time_selector = '#hourly-forecast-table > tbody > tr:nth-child(1) 
    > td.mat-cell.cdk-cell.cdk-column-timeHour.mat-column- 
    timeHour.ng-star-inserted > span'
    temperature_selector = '#hourly-forecast-table > tbody > tr:nth- 
    child(1) > td.mat-cell.cdk-cell.cdk-column-temperature.mat- 
    column-temperature.ng-star-inserted'
    humidity_selector = '#hourly-forecast-table > tbody > tr:nth- 
    child(1) > td.mat-cell.cdk-cell.cdk-column-humidity.mat-column- 
    humidity.ng-star-inserted'
    precip_selector = '#hourly-forecast-table > tbody > tr:nth- 
    child(1) > td.mat-cell.cdk-cell.cdk-column-precipitation.mat- 
    column-precipitation.ng-star-inserted'
    wind_speed_selector = '#hourly-forecast-table > tbody > tr:nth- 
    child(1) > td.mat-cell.cdk-cell.cdk-column-wind.mat-column- 
    wind.ng-star-inserted'
    pressure_selector = '#hourly-forecast-table > tbody > tr:nth- 
    child(1) > td.mat-cell.cdk-cell.cdk-column-pressure.mat-column- 
    pressure.ng-star-inserted'
    amount_selector = '#hourly-forecast-table > tbody > tr:nth- 
    child(1) > td.mat-cell.cdk-cell.cdk-column- 
    liquidPrecipitation.mat-column-liquidPrecipitation.ng-star- 
    inserted'


    # Calling a function and saving the Data
    scrape_weather_data(url, time_selector, temperature_selector, 
    humidity_selector, precip_selector, wind_speed_selector, 
    pressure_selector, amount_selector)

So basically the most important 2 lines are :

 driver.find_element(By.CSS_SELECTOR, '#wuSettings > i').click()

 driver.find_element(By.CSS_SELECTOR, '#wuSettings-quick > div > 
 a:nth-child(2)').click()

The first one is responsible to enter the settings[the gear icon] (which from where we can change the measurements systems), and the other one is responsible to click the 2nd option which is the metric system.

The problem is that when i try do that the output in my textfile is "------------------------------". I don't know why. It seems that is only working with selectors, that don't have such a path "a:nth-child". Because when I change that to celsius, then the imperial system becomes the selector that is"#wuSettings-quick > div > a:nth-child(1)". And the chosen selector is "#wuSettings-quick > div > a.button.selected" which works perfectly fine.

How do I solve that?

Answer 1

Observe the <div class="button-group"> element

There are descendent pseudo elements ::before and ::after

---

Solution

To click on °C you need to induce WebDriverWait for the visibility_of_element_located() of the <table> element first and then inducing WebDriverWait for the element_to_be_clickable() you can use the following locator strategies:

• Using CSS_SELECTOR:

  driver.get("https://www.wunderground.com/hourly/pl/kucoby")
  WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "table#hourly-forecast-table")))
  WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#wuSettings > i"))).click()
  WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#wuSettings-quick div a:nth-child(2)"))).click()
  

• Note: You have to add the following imports :

  from selenium.webdriver.support.ui import WebDriverWait
  from selenium.webdriver.common.by import By
  from selenium.webdriver.support import expected_conditions as EC