How to get all child elements using Selenium

Question

I am trying to grab all the current teams and expected outcomes for a nflpickwatch.com. I am using Selenium to grab the info since it's using JavaScript.

If you go the site site above and inspect the elements you will see each persons info is under "row-yellow" or "row-yellow-dark." My question is how would I go about getting each persons name and who they choose in a dict.

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait

info = {}
# -------------- Finds the website --------------- #
website = "https://nflpickwatch.com/nfl/picks/su/experts"
service = Service("C:\Program Files\Mozilla Firefox\geckodriver.exe")
driver = webdriver.Firefox(service=service)
driver.maximize_window()
driver.implicitly_wait(15)
driver.get(website)

# ----------- Closes the ad --------- #
driver.switch_to.frame(driver.find_element(By.XPATH, '//iframe[@class="intercom-1mubrwr e1ctqele0"]'))
test = driver.find_element(By.XPATH, '//span[@aria-label="Close"]')
test.click()
driver.switch_to.default_content()


# ------- Finds the players and get info ------- #
name = driver.find_element(By.XPATH, '//tr[@class="row-yellow" or @class="row-yellow-dark"]')


# ------- Finds the Game they expect to win ---------- #

game = driver.find_element(By.XPATH, '//tbody[@role="rowgroup"]')
game2 = game.find_elements(By.XPATH, "//div[@class='rounded transparent-border']")
for i in game2:
    print(i.get_attribute('title'))
driver.close()

This is my current code and when you run it, it just gets a list of every currenlt choice from everyone. My desiered outcome would be info = {f"{Person name}": "Projected winnner": ""}

Answer 1

Don't use Selenium. The data comes from the api in json (dictionary) form already:

import requests
import pandas as pd

nfl_week = 3

url = f'https://api.nflpickwatch.com/v1/nfl/picks/2022/{nfl_week}/su/experts/true/999/0'

jsonData = requests.get(url).json()
df = pd.json_normalize(jsonData['picks'])

Output: just showing you the first 3 elements here

print(jsonData['picks'][0:3])
[
 {'row_count': 662, 'user_id': 297, 'username': 'Colleen Wolfe', 'avatar_key': 'avatars/users/297/2971661164629615.png', 'is_pro': False, 'affiliation': 'NFL', 'affiliate_avatar': 'networks/logos/43/431660297150903.png', 'rank': 1, 'wins': 1, 'losses': 0, 'win_pct': 100, 'pick_pct': 100, 'profit_loss': 25, 'season_wins': 22, 'season_losses': 10, 'season_win_pct': 68.75, 'season_pick_pct': 100, 'season_profit_loss': 97.5, 'prev_season_wins': 180, 'prev_season_losses': 104, 'prev_season_win_pct': 63.38, 'prev_season_pick_pct': 100, 'prev_season_profit_loss': -353.04, 'interval_wins': 22, 'interval_losses': 10, 'interval_win_pct': 66.66666666666667, 'interval_pick_pct': 100, 'interval_profit_loss': 0, 'underdog_wins': 0, 'underdog_losses': 0, 'underdog_win_pct': 0, 'underdog_pick_pct': 0, 'underdog_profit_loss': 0, 'underdog_season_wins': 0, 'underdog_season_losses': 0, 'underdog_season_win_pct': 0, 'underdog_season_pick_pct': 0, 'underdog_season_profit_loss': 0, 'underdog_prev_season_wins': 0, 'underdog_prev_season_losses': 0, 'underdog_prev_season_win_pct': 0, 'underdog_prev_season_pick_pct': 0, 'underdog_prev_season_profit_loss': 0, 'underdog_interval_wins': 0, 'underdog_interval_losses': 0, 'underdog_interval_win_pct': 0, 'underdog_interval_pick_pct': 0, 'identifier': 'colleen-wolfe', 'picks': {'2200': {'stake': None, 'underdog': None, 'seq': 4, 'team_id': 'KC', 'result': 'pregame'}, '2201': {'stake': None, 'underdog': None, 'seq': 7, 'team_id': 'BAL', 'result': 'pregame'}, '2202': {'stake': None, 'underdog': None, 'seq': 14, 'team_id': 'GB', 'result': 'pregame'}, '2203': {'stake': 50, 'underdog': None, 'seq': 1, 'team_id': 'CLE', 'result': 'win'}, '2204': {'stake': None, 'underdog': None, 'seq': 2, 'team_id': 'CHI', 'result': 'pregame'}, '2205': {'stake': None, 'underdog': None, 'seq': 3, 'team_id': 'LV', 'result': 'pregame'}, '2206': {'stake': None, 'underdog': None, 'seq': 5, 'team_id': 'BUF', 'result': 'pregame'}, '2207': {'stake': None, 'underdog': None, 'seq': 6, 'team_id': 'DET', 'result': 'pregame'}, '2208': {'stake': None, 'underdog': None, 'seq': 8, 'team_id': 'CIN', 'result': 'pregame'}, '2209': {'stake': None, 'underdog': None, 'seq': 9, 'team_id': 'PHI', 'result': 'pregame'}, '2210': {'stake': None, 'underdog': None, 'seq': 10, 'team_id': 'NO', 'result': 'pregame'}, '2211': {'stake': None, 'underdog': None, 'seq': 11, 'team_id': 'LAC', 'result': 'pregame'}, '2212': {'stake': None, 'underdog': None, 'seq': 12, 'team_id': 'LAR', 'result': 'pregame'}, '2213': {'stake': None, 'underdog': None, 'seq': 13, 'team_id': 'ATL', 'result': 'pregame'}, '2214': {'stake': None, 'underdog': None, 'seq': 15, 'team_id': 'DEN', 'result': 'pregame'}, '2215': {'stake': None, 'underdog': None, 'seq': 16, 'team_id': 'NYG', 'result': 'pregame'}}},
 {'row_count': 662, 'user_id': 185, 'username': 'Craig Miller', 'avatar_key': 'avatars/users/185/1851660838014733.png', 'is_pro': False, 'affiliation': 'DallasMorningNews', 'affiliate_avatar': 'networks/logos/3/31607611975864.png', 'rank': 1, 'wins': 1, 'losses': 0, 'win_pct': 100, 'pick_pct': 100, 'profit_loss': 0, 'season_wins': 22, 'season_losses': 10, 'season_win_pct': 68.75, 'season_pick_pct': 100, 'season_profit_loss': 79, 'prev_season_wins': 181, 'prev_season_losses': 103, 'prev_season_win_pct': 63.732, 'prev_season_pick_pct': 100, 'prev_season_profit_loss': -1028.49, 'interval_wins': 22, 'interval_losses': 10, 'interval_win_pct': 66.66666666666667, 'interval_pick_pct': 100, 'interval_profit_loss': 0, 'underdog_wins': 0, 'underdog_losses': 0, 'underdog_win_pct': 0, 'underdog_pick_pct': 0, 'underdog_profit_loss': 0, 'underdog_season_wins': 0, 'underdog_season_losses': 0, 'underdog_season_win_pct': 0, 'underdog_season_pick_pct': 0, 'underdog_season_profit_loss': 0, 'underdog_prev_season_wins': 0, 'underdog_prev_season_losses': 0, 'underdog_prev_season_win_pct': 0, 'underdog_prev_season_pick_pct': 0, 'underdog_prev_season_profit_loss': 0, 'underdog_interval_wins': 0, 'underdog_interval_losses': 0, 'underdog_interval_win_pct': 0, 'underdog_interval_pick_pct': 0, 'identifier': 'craig-miller', 'picks': {'2200': {'stake': None, 'underdog': None, 'seq': 4, 'team_id': 'KC', 'result': 'pregame'}, '2201': {'stake': None, 'underdog': None, 'seq': 7, 'team_id': 'BAL', 'result': 'pregame'}, '2202': {'stake': None, 'underdog': None, 'seq': 14, 'team_id': 'TB', 'result': 'pregame'}, '2203': {'stake': 50, 'underdog': None, 'seq': 1, 'team_id': 'CLE', 'result': 'win'}, '2204': {'stake': None, 'underdog': None, 'seq': 2, 'team_id': 'CHI', 'result': 'pregame'}, '2205': {'stake': None, 'underdog': None, 'seq': 3, 'team_id': 'TEN', 'result': 'pregame'}, '2206': {'stake': None, 'underdog': None, 'seq': 5, 'team_id': 'BUF', 'result': 'pregame'}, '2207': {'stake': None, 'underdog': None, 'seq': 6, 'team_id': 'MIN', 'result': 'pregame'}, '2208': {'stake': None, 'underdog': None, 'seq': 8, 'team_id': 'CIN', 'result': 'pregame'}, '2209': {'stake': None, 'underdog': None, 'seq': 9, 'team_id': 'PHI', 'result': 'pregame'}, '2210': {'stake': None, 'underdog': None, 'seq': 10, 'team_id': 'NO', 'result': 'pregame'}, '2211': {'stake': None, 'underdog': None, 'seq': 11, 'team_id': 'LAC', 'result': 'pregame'}, '2212': {'stake': None, 'underdog': None, 'seq': 12, 'team_id': 'LAR', 'result': 'pregame'}, '2213': {'stake': None, 'underdog': None, 'seq': 13, 'team_id': 'SEA', 'result': 'pregame'}, '2214': {'stake': None, 'underdog': None, 'seq': 15, 'team_id': 'SF', 'result': 'pregame'}, '2215': {'stake': None, 'underdog': None, 'seq': 16, 'team_id': 'NYG', 'result': 'pregame'}}},
 {'row_count': 662, 'user_id': 77080, 'username': 'Domonique Foxworth', 'avatar_key': None, 'is_pro': False, 'affiliation': 'ESPN', 'affiliate_avatar': 'networks/logos/5/51660839451660.png', 'rank': 1, 'wins': 1, 'losses': 0, 'win_pct': 100, 'pick_pct': 100, 'profit_loss': 26.32, 'season_wins': 22, 'season_losses': 10, 'season_win_pct': 68.75, 'season_pick_pct': 100, 'season_profit_loss': 327.45, 'prev_season_wins': 174, 'prev_season_losses': 109, 'prev_season_win_pct': 61.484, 'prev_season_pick_pct': 99.649, 'prev_season_profit_loss': -881.97, 'interval_wins': 22, 'interval_losses': 10, 'interval_win_pct': 66.66666666666667, 'interval_pick_pct': 100, 'interval_profit_loss': 0, 'underdog_wins': 0, 'underdog_losses': 0, 'underdog_win_pct': 0, 'underdog_pick_pct': 0, 'underdog_profit_loss': 0, 'underdog_season_wins': 0, 'underdog_season_losses': 0, 'underdog_season_win_pct': 0, 'underdog_season_pick_pct': 0, 'underdog_season_profit_loss': 0, 'underdog_prev_season_wins': 0, 'underdog_prev_season_losses': 0, 'underdog_prev_season_win_pct': 0, 'underdog_prev_season_pick_pct': 0, 'underdog_prev_season_profit_loss': 0, 'underdog_interval_wins': 0, 'underdog_interval_losses': 0, 'underdog_interval_win_pct': 0, 'underdog_interval_pick_pct': 0, 'identifier': 'domonique-foxworth', 'picks': {'2200': {'stake': None, 'underdog': None, 'seq': 4, 'team_id': 'KC', 'result': 'pregame'}, '2201': {'stake': None, 'underdog': None, 'seq': 7, 'team_id': 'BAL', 'result': 'pregame'}, '2202': {'stake': None, 'underdog': None, 'seq': 14, 'team_id': 'TB', 'result': 'pregame'}, '2203': {'stake': 50, 'underdog': None, 'seq': 1, 'team_id': 'CLE', 'result': 'win'}, '2204': {'stake': None, 'underdog': None, 'seq': 2, 'team_id': 'HOU', 'result': 'pregame'}, '2205': {'stake': None, 'underdog': None, 'seq': 3, 'team_id': 'LV', 'result': 'pregame'}, '2206': {'stake': None, 'underdog': None, 'seq': 5, 'team_id': 'MIA', 'result': 'pregame'}, '2207': {'stake': None, 'underdog': None, 'seq': 6, 'team_id': 'MIN', 'result': 'pregame'}, '2208': {'stake': None, 'underdog': None, 'seq': 8, 'team_id': 'CIN', 'result': 'pregame'}, '2209': {'stake': None, 'underdog': None, 'seq': 9, 'team_id': 'PHI', 'result': 'pregame'}, '2210': {'stake': None, 'underdog': None, 'seq': 10, 'team_id': 'NO', 'result': 'pregame'}, '2211': {'stake': None, 'underdog': None, 'seq': 11, 'team_id': 'LAC', 'result': 'pregame'}, '2212': {'stake': None, 'underdog': None, 'seq': 12, 'team_id': 'LAR', 'result': 'pregame'}, '2213': {'stake': None, 'underdog': None, 'seq': 13, 'team_id': 'SEA', 'result': 'pregame'}, '2214': {'stake': None, 'underdog': None, 'seq': 15, 'team_id': 'SF', 'result': 'pregame'}, '2215': {'stake': None, 'underdog': None, 'seq': 16, 'team_id': 'NYG', 'result': 'pregame'}}}
]

Answer 2

based on what i understood from your question, try:

import pandas as pd
from bs4 import BeautifulSoup

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service               #used to give driver path
from selenium.webdriver.common.keys import Keys                     #used for keyboard keys 
from selenium.webdriver.support.ui import WebDriverWait             #used for waiting elem
from selenium.webdriver.support import expected_conditions as EC    #used for waiting elem 


options = Options()
# options.add_argument("--headless")
options.add_argument('log-level=3')
options.add_argument("--disable-notifications")
options.add_argument("--start-maximized")
options.add_argument('--disable-blink-features=AutomationControlled')
options.add_experimental_option("excludeSwitches", ["enable-automation"]) 
options.add_experimental_option('excludeSwitches', ['enable-logging']) 
options.add_experimental_option('useAutomationExtension', False)

driver = webdriver.Chrome(service=Service("chromedriver.exe"), options=options) 
driver.get("https://nflpickwatch.com/nfl/picks/su/experts")
driver.implicitly_wait(60) 

#------close the ad-----------
WebDriverWait(driver, 10).until(EC.frame_to_be_available_and_switch_to_it((By.XPATH, '//iframe[@class="intercom-1mubrwr e1ctqele0"]')))
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//span[@aria-label="Close"]'))).click()

driver.switch_to.default_content() #comeback to main html // exit iframe

#---------scrape the data--------
soup = BeautifulSoup(driver.page_source, 'html.parser')

ROWS = []
for i in soup.select('.row-yellow, .row-yellow-dark'):
    rank = i.select_one('td:nth-of-type(1)').text
    pname = i.select_one('td:nth-of-type(2) a').text.strip()

    pick1 = i.select_one('td:nth-of-type(3) div > div > div')['title'] if i.select_one('td:nth-of-type(3) div > div > div') else None
    pick2 = i.select_one('td:nth-of-type(4) div > div > div')['title'] if i.select_one('td:nth-of-type(4) div > div > div') else None
    pick3 = i.select_one('td:nth-of-type(5) div > div > div')['title'] if i.select_one('td:nth-of-type(5) div > div > div') else None
    pick4 = i.select_one('td:nth-of-type(6) div > div > div')['title'] if i.select_one('td:nth-of-type(6) div > div > div') else None
    pick5 = i.select_one('td:nth-of-type(7) div > div > div')['title'] if i.select_one('td:nth-of-type(7) div > div > div') else None
    pick6 = i.select_one('td:nth-of-type(8) div > div > div')['title'] if i.select_one('td:nth-of-type(8) div > div > div') else None
    pick7 = i.select_one('td:nth-of-type(9) div > div > div')['title'] if i.select_one('td:nth-of-type(9) div > div > div') else None
    pick8 = i.select_one('td:nth-of-type(10) div > div > div')['title'] if i.select_one('td:nth-of-type(10) div > div > div') else None
    pick9 = i.select_one('td:nth-of-type(11) div > div > div')['title'] if i.select_one('td:nth-of-type(11) div > div > div') else None
    pick10 = i.select_one('td:nth-of-type(12) div > div > div')['title'] if i.select_one('td:nth-of-type(12) div > div > div') else None
    pick11 = i.select_one('td:nth-of-type(13) div > div > div')['title'] if i.select_one('td:nth-of-type(13) div > div > div') else None
    pick12 = i.select_one('td:nth-of-type(14) div > div > div')['title'] if i.select_one('td:nth-of-type(14) div > div > div') else None
    pick13 = i.select_one('td:nth-of-type(15) div > div > div')['title'] if i.select_one('td:nth-of-type(15) div > div > div') else None
    pick14 = i.select_one('td:nth-of-type(16) div > div > div')['title'] if i.select_one('td:nth-of-type(16) div > div > div') else None
    pick15 = i.select_one('td:nth-of-type(17) div > div > div')['title'] if i.select_one('td:nth-of-type(17) div > div > div') else None
    pick16 = i.select_one('td:nth-of-type(18) div > div > div')['title'] if i.select_one('td:nth-of-type(18) div > div > div') else None

    wins_this_week = i.select_one('td:nth-of-type(19)').text.strip()
    losses_this_week = i.select_one('td:nth-of-type(20)').text.strip()
    w_perc_this_week = i.select_one('td:nth-of-type(21)').text.strip()
    wins_this_season = i.select_one('td:nth-of-type(22)').text.strip()
    losses_this_season = i.select_one('td:nth-of-type(23)').text.strip()
    w_perc_this_season = i.select_one('td:nth-of-type(24)').text.strip()
    row = [pname, rank, wins_this_week, losses_this_week, w_perc_this_week, wins_this_season, losses_this_season, w_perc_this_season, pick1, pick2, pick3, pick4, pick5, pick6, pick7, pick8, pick9, pick10, pick11, pick12, pick13, pick14, pick15, pick16,]
    ROWS.append(row)

#will return ROWS: a list of lists (each row in a list) 

#----------convert ROWS to dictionary, as you asked--------------------------
D = dict(zip([i[0] for i in ROWS], [i[1:] for i in ROWS]))

#----------convert ROWS to dataframe--------------------------

df = pd.DataFrame(ROWS)

Answer 3

I like turning my webelements into easy to use classes, so I would recommend something like this.

class Row:

    def __init__(self, web_element):
        self._element = web_element


    @property
    def name(self) -> str:
        return self._element.find_element(By.CSS_SELECTOR, '.name-cell__name').text

    @property
    def top_pick(self) -> str:
        return self._element.find_element(By.CSS_SELECTOR, '.logo-border').get_attribute('title')


rows = map(Row, driver.get_elements(By.CSS_SELECTOR, 'tbody tr'))

dictionary_to_return = {}
for row in rows[1:]:
    dictionary_to_return[row.name] = row.top_pick

print(dictionary_to_return)

I should probably clarify a bit, WebElements have almost the same interface as the driver, so many of the same driver methods can be called on the WebElement, such as find_elements. But using these methods on a WebElement will only use the scope of the WebElement.

So by using the find_elements/find_element method on a WebElement it will only return child elements.