erselbal
erselbal

Reputation: 17

HTTPConnectionPool Max retries exceeded with url" Error in Python Selenium

I have a web application where users can perform searches on specific retailers and view the results in tables. When two users start searching on t:he same retailer at the same time, after a while, I get an error:

HTTPConnectionPool(host='localhost', port=57640): Max retries exceeded with url: /session/9b99c06c009ea88cba5295bd6fb3725a/url (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x0000021B3ED66310>: Failed to establish a new connection: [WinError 10061]))

Sometimes I also get

InvalidSessionIdException at /search/

If the retailers being searched are not the same, the program works fine. It also works fine if the searches don't start or end at almost the same time.

Chrome: 114.0.5735.110 Selenium: 4.9.1 Django: 4.2.1

Python Selenium Code:

   def standard_analysis(self, product_keys:list[str], category:str,search_method:int=0, img_method:int=0, goal:int=50, sleep_time:float=1):
        """
        It searches for the given product codes in the given retails and analyzes the information it finds.
        * product_keys: The products from which information will be obtained in the search and analysis should be given as a list of strings.
        * category:  Category type of products to be searched.
        -----------
        * search_method '0': Google search with Request library
        * search_method '1': Google search with Google library 
        * search_method '2': Google search with Selenium Webdriver
        * search_method '3': Search in Retail
        -----------
        * img_method '0': Take screenshot
        * img_method '1': Download image
        -----------
        * goal: The expected target score to be obtained as a result of the analysis.
        * sleep_time: The waiting time between operations.
        """
        chrome_options = Options()
        chrome_options.add_argument(f"--remote-debugging-port=0")
        results=[]
        for retail in self.retails:
            if category.upper() not in retail['categories']:
                raise CategoryException(category, retail['name'])

            searcher = ProductSearch(retail['url'], product_keys)

            query_result = searcher.get_links(search_method, sleep_time=sleep_time,retail_rules=retail['retail_rules'])
            links = query_result[0]

            h = retail['parser']
            h.driver = webdriver.Chrome(options=chrome_options)
            print(f'Port: {h.driver.service.service_url}')
            for l in links:
                key = l[0]
                if l[1] == None:
                    continue

                h.driver.get(l[1])
                h.driver.maximize_window()
                if retail['retail_rules']['cookie_btn'] and h.is_exist(retail['retail_rules']['cookie_btn']):
                    cookie_btn = h.driver.find_element(By.XPATH, retail['retail_rules']['cookie_btn'])
                    cookie_btn.send_keys(Keys.ENTER)

                product_rules = retail['product_rules']
                product_ver = retail['product_verification']

                if img_method > 1:
                    img_method = 0
                if img_method == 0:
                    h.screenshot_image(product_rules, replace_chars(key))
                if img_method == 1:
                    h.download_image(product_rules, replace_chars(key))

                verify_lvl = h.get_verif_lvl(product_rules=product_rules, product_ver=product_ver, product_key=key, category=category)
    
                result = h.get_all_info(category.upper(), retail['name'], key, l[1], verify_lvl, product_rules) 
                results.append(result)
            h.driver.quit()

        return results

Django Views:

@login_required(login_url='index')
def search(request):
    form = checkform(request)
    if request.method == 'GET':
        products = request.session.get('products', [])
        return render(request, 'search.html', {'products': products, 'rform': form,'retailers': RETAILS, 'categories': CATEGORIES})
    if request.method == 'POST':
        action = request.POST.get('action')
        if action == 'Add':
            excel_file = request.FILES.get('excel_file')
            if excel_file:
                if excel_file.content_type == 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
                    df = pd.read_excel(excel_file, engine='openpyxl')
                    column_name = 'Products'
                    products = df[column_name].astype(str).values.tolist()
                    request.session['products'] = products
                    return render(request, 'search.html', {'products': products, 'rform': form,'retailers': RETAILS, 'categories': CATEGORIES})
                else:
                    products = request.session.get('products', [])
                    error="File is not xls"
                    return render(request, 'search.html', {'products': products, 'rform': form,'error':error,'retailers': RETAILS, 'categories': CATEGORIES})
            else:
                products = request.session.get('products', [])
                error="No file"
                return render(request, 'search.html', {'products': products, 'rform': form,'error':error,'retailers': RETAILS, 'categories': CATEGORIES})
        elif action == 'Clear':
            products=[]
            request.session['products'] = products
            return render(request, 'search.html', {'products': products, 'rform': form,'retailers': RETAILS, 'categories': CATEGORIES})
        elif action == 'Send':
            
            araform = retailForm(request.POST)
            category = request.POST["category"]
            if araform.is_valid():
                sites = (araform.cleaned_data.get('sites'))
                retails = []
                for sit in sites:
                    if sit == 'Trendyol':
                        retails.append(r.TRENDYOL)

                    elif sit == 'Emagro':
                        retails.append(r.EMAGRO)

                    elif sit == 'Emagbg':
                        retails.append(r.EMAGBG)

                    elif sit == 'Gigatron':
                        retails.append(r.GIGATRON)

                    elif sit == 'Coolshop':
                        retails.append(r.COOLSHOP)

                    elif sit == 'Amazon':
                        retails.append(r.AMAZON)

                    elif sit == 'Tehnomanija':
                        retails.append(r.TEHNOMANIJA)

                    elif sit == 'Altexro':
                        retails.append(r.ALTEXRO)

                    elif sit == 'Elbraco':
                        retails.append(r.ELBRACO)
                
                tool = Analyzer(retails)
                with transaction.atomic():
                    products = request.session.get('products', [])
                    result = tool.standard_analysis(products, category=category)

                if request.user.is_authenticated:
                    pk = request.POST['rate']
                    grid = Grid.objects.create(user=request.user,name=request.POST['name'])
                    for res in result:
                        total = calc_rate(res, pk)
                        res['rate'] = total
                        Item.objects.create(
                            grid_Item=grid,
                            retail= res['retail'],
                            key= res['key'],
                            name= res['name'],
                            stock_info= res['stock'],
                            price= True if res['price'] is not None and res['price'] != 0 else False,
                            disc= True if res['disc_price'] is not None and res['disc_price'] != 0 else False,
                            desc= True if res['description'] is not None else False,
                            manual= res['manual'] if res['manual'] is not None else False,
                            image= res['image_count'],
                            video= True if res['video'] is not None else False,
                            rich_content= res['rich_content'] if res['rich_content'] is not None else False,
                            energy= True if res['energy_label'] is not None else False,
                            review= True if res['review_count'] is not None and res['review_count'] != 0 else False,
                            verif_lvl= res['verification'],
                            url= res['link'],
                            ai= False,
                            rate= res['rate'],
                        )
                    return render(request, 'grid.html', {'result': result})
                else:
                    return render(request, 'grid.html', {'result': result})

Upvotes: 0

Views: 962

Answers (0)

Related Questions