Reputation: 17
I have a web application where users can perform searches on specific retailers and view the results in tables. When two users start searching on t:he same retailer at the same time, after a while, I get an error:
HTTPConnectionPool(host='localhost', port=57640): Max retries exceeded with url: /session/9b99c06c009ea88cba5295bd6fb3725a/url (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x0000021B3ED66310>: Failed to establish a new connection: [WinError 10061]))
Sometimes I also get
InvalidSessionIdException at /search/
If the retailers being searched are not the same, the program works fine. It also works fine if the searches don't start or end at almost the same time.
Chrome: 114.0.5735.110 Selenium: 4.9.1 Django: 4.2.1
Python Selenium Code:
def standard_analysis(self, product_keys:list[str], category:str,search_method:int=0, img_method:int=0, goal:int=50, sleep_time:float=1):
"""
It searches for the given product codes in the given retails and analyzes the information it finds.
* product_keys: The products from which information will be obtained in the search and analysis should be given as a list of strings.
* category: Category type of products to be searched.
-----------
* search_method '0': Google search with Request library
* search_method '1': Google search with Google library
* search_method '2': Google search with Selenium Webdriver
* search_method '3': Search in Retail
-----------
* img_method '0': Take screenshot
* img_method '1': Download image
-----------
* goal: The expected target score to be obtained as a result of the analysis.
* sleep_time: The waiting time between operations.
"""
chrome_options = Options()
chrome_options.add_argument(f"--remote-debugging-port=0")
results=[]
for retail in self.retails:
if category.upper() not in retail['categories']:
raise CategoryException(category, retail['name'])
searcher = ProductSearch(retail['url'], product_keys)
query_result = searcher.get_links(search_method, sleep_time=sleep_time,retail_rules=retail['retail_rules'])
links = query_result[0]
h = retail['parser']
h.driver = webdriver.Chrome(options=chrome_options)
print(f'Port: {h.driver.service.service_url}')
for l in links:
key = l[0]
if l[1] == None:
continue
h.driver.get(l[1])
h.driver.maximize_window()
if retail['retail_rules']['cookie_btn'] and h.is_exist(retail['retail_rules']['cookie_btn']):
cookie_btn = h.driver.find_element(By.XPATH, retail['retail_rules']['cookie_btn'])
cookie_btn.send_keys(Keys.ENTER)
product_rules = retail['product_rules']
product_ver = retail['product_verification']
if img_method > 1:
img_method = 0
if img_method == 0:
h.screenshot_image(product_rules, replace_chars(key))
if img_method == 1:
h.download_image(product_rules, replace_chars(key))
verify_lvl = h.get_verif_lvl(product_rules=product_rules, product_ver=product_ver, product_key=key, category=category)
result = h.get_all_info(category.upper(), retail['name'], key, l[1], verify_lvl, product_rules)
results.append(result)
h.driver.quit()
return results
Django Views:
@login_required(login_url='index')
def search(request):
form = checkform(request)
if request.method == 'GET':
products = request.session.get('products', [])
return render(request, 'search.html', {'products': products, 'rform': form,'retailers': RETAILS, 'categories': CATEGORIES})
if request.method == 'POST':
action = request.POST.get('action')
if action == 'Add':
excel_file = request.FILES.get('excel_file')
if excel_file:
if excel_file.content_type == 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
df = pd.read_excel(excel_file, engine='openpyxl')
column_name = 'Products'
products = df[column_name].astype(str).values.tolist()
request.session['products'] = products
return render(request, 'search.html', {'products': products, 'rform': form,'retailers': RETAILS, 'categories': CATEGORIES})
else:
products = request.session.get('products', [])
error="File is not xls"
return render(request, 'search.html', {'products': products, 'rform': form,'error':error,'retailers': RETAILS, 'categories': CATEGORIES})
else:
products = request.session.get('products', [])
error="No file"
return render(request, 'search.html', {'products': products, 'rform': form,'error':error,'retailers': RETAILS, 'categories': CATEGORIES})
elif action == 'Clear':
products=[]
request.session['products'] = products
return render(request, 'search.html', {'products': products, 'rform': form,'retailers': RETAILS, 'categories': CATEGORIES})
elif action == 'Send':
araform = retailForm(request.POST)
category = request.POST["category"]
if araform.is_valid():
sites = (araform.cleaned_data.get('sites'))
retails = []
for sit in sites:
if sit == 'Trendyol':
retails.append(r.TRENDYOL)
elif sit == 'Emagro':
retails.append(r.EMAGRO)
elif sit == 'Emagbg':
retails.append(r.EMAGBG)
elif sit == 'Gigatron':
retails.append(r.GIGATRON)
elif sit == 'Coolshop':
retails.append(r.COOLSHOP)
elif sit == 'Amazon':
retails.append(r.AMAZON)
elif sit == 'Tehnomanija':
retails.append(r.TEHNOMANIJA)
elif sit == 'Altexro':
retails.append(r.ALTEXRO)
elif sit == 'Elbraco':
retails.append(r.ELBRACO)
tool = Analyzer(retails)
with transaction.atomic():
products = request.session.get('products', [])
result = tool.standard_analysis(products, category=category)
if request.user.is_authenticated:
pk = request.POST['rate']
grid = Grid.objects.create(user=request.user,name=request.POST['name'])
for res in result:
total = calc_rate(res, pk)
res['rate'] = total
Item.objects.create(
grid_Item=grid,
retail= res['retail'],
key= res['key'],
name= res['name'],
stock_info= res['stock'],
price= True if res['price'] is not None and res['price'] != 0 else False,
disc= True if res['disc_price'] is not None and res['disc_price'] != 0 else False,
desc= True if res['description'] is not None else False,
manual= res['manual'] if res['manual'] is not None else False,
image= res['image_count'],
video= True if res['video'] is not None else False,
rich_content= res['rich_content'] if res['rich_content'] is not None else False,
energy= True if res['energy_label'] is not None else False,
review= True if res['review_count'] is not None and res['review_count'] != 0 else False,
verif_lvl= res['verification'],
url= res['link'],
ai= False,
rate= res['rate'],
)
return render(request, 'grid.html', {'result': result})
else:
return render(request, 'grid.html', {'result': result})
Upvotes: 0
Views: 962