Reputation: 77
I have a simple code that scrapes the Polish betting site and stores data in the csv file. I would like to have market names translated from Polish to English in csv file. My current solution is not working, instead, it opens a new instance of chrome with an empty page, and my code breaks. Here is my code:
from os import pardir
import time
from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from urllib.parse import urlparse, parse_qs
import re
import pandas as pd
import numpy as np
from selenium.webdriver.remote.webelement import BaseWebElement
from googletrans import Translator
translator = Translator()
driver = webdriver.Chrome('C:/Users/peroperic/source/repos/chromedriver.exe')
team_name = 'Gabala 2'
u =f'https://superbet.pl/wyszukaj?query={team_name}'
url = driver.get(u)
driver.maximize_window()
time.sleep(1)
driver.find_element(By.XPATH,'//*[@id="onetrust-accept-btn-handler"]').click()
time.sleep(1)
driver.find_element(By.CLASS_NAME,'pick__more-odds').click()
time.sleep(3)
######################################################### - this part of code should translate the page but is not working
options = webdriver.ChromeOptions()
prefs = {
"translate_whitelists": {"po":"en"},
"translate":{"enabled":"True"}
}
options.add_experimental_option('prefs', prefs)
driver = webdriver.Chrome('C:/Users/peroperic/source/repos/chromedriver.exe',chrome_options=options)
#########################################################
expand = driver.find_elements(By.CLASS_NAME,'icon.icon--md.event-row__expanded-market-icon.icon-chevron_down')
df = pd.DataFrame()
try:
exp_clicks = 0
for i in expand:
i.click()
time.sleep(0.2)
exp_clicks +=1
time.sleep(1)
market_data = []
market_list = driver.find_elements(By.CLASS_NAME,'event-row__expanded-market')
for market in market_list:
counter = 0
name_elem = market.find_element(By.CLASS_NAME,'event-row__expanded-market-title')
name = name_elem.text
market_data.append(name)
price_element = market.find_elements(By.CLASS_NAME,'value.new.actionable')
for j in price_element:
price_list = []
price = j.text
market_data.append(price)
market_df = pd.DataFrame(np.array(market_data).reshape(-1,len(market_data)))
df = pd.concat([df,market_df], ignore_index = True, axis = 0).replace(np.nan,'')
market_data.clear()
counter += 1
except:
pass
df.to_csv('C:/Users/peroperic/source/repos/Sportsbook WebCsrapper/Data.csv')
I browsed for answers on this topic with no luck. Any idea why this is not working for me? Thank you.
Upvotes: 0
Views: 290
Reputation: 33351
You should put the following block
options = webdriver.ChromeOptions()
prefs = {
"translate_whitelists": {"po":"en"},
"translate":{"enabled":"True"}
}
options.add_experimental_option('prefs', prefs)
driver = webdriver.Chrome('C:/Users/peroperic/source/repos/chromedriver.exe',chrome_options=options)
before the first creation of driver
instance with this
driver = webdriver.Chrome('C:/Users/peroperic/source/repos/chromedriver.exe')
Try this:
from os import pardir
import time
from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from urllib.parse import urlparse, parse_qs
import re
import pandas as pd
import numpy as np
from selenium.webdriver.remote.webelement import BaseWebElement
from googletrans import Translator
translator = Translator()
#########################################################
options = webdriver.ChromeOptions()
prefs = {
"translate_whitelists": {"po":"en"},
"translate":{"enabled":"True"}
}
options.add_experimental_option('prefs', prefs)
driver = webdriver.Chrome('C:/Users/peroperic/source/repos/chromedriver.exe',chrome_options=options)
#########################################################
team_name = 'Gabala 2'
u =f'https://superbet.pl/wyszukaj?query={team_name}'
url = driver.get(u)
driver.maximize_window()
time.sleep(1)
driver.find_element(By.XPATH,'//*[@id="onetrust-accept-btn-handler"]').click()
time.sleep(1)
driver.find_element(By.CLASS_NAME,'pick__more-odds').click()
time.sleep(3)
expand = driver.find_elements(By.CLASS_NAME,'icon.icon--md.event-row__expanded-market-icon.icon-chevron_down')
df = pd.DataFrame()
try:
exp_clicks = 0
for i in expand:
i.click()
time.sleep(0.2)
exp_clicks +=1
time.sleep(1)
market_data = []
market_list = driver.find_elements(By.CLASS_NAME,'event-row__expanded-market')
for market in market_list:
counter = 0
name_elem = market.find_element(By.CLASS_NAME,'event-row__expanded-market-title')
name = name_elem.text
market_data.append(name)
price_element = market.find_elements(By.CLASS_NAME,'value.new.actionable')
for j in price_element:
price_list = []
price = j.text
market_data.append(price)
market_df = pd.DataFrame(np.array(market_data).reshape(-1,len(market_data)))
df = pd.concat([df,market_df], ignore_index = True, axis = 0).replace(np.nan,'')
market_data.clear()
counter += 1
except:
pass
df.to_csv('C:/Users/peroperic/source/repos/Sportsbook WebCsrapper/Data.csv')
Upvotes: 1