Playwright with python - Download file from CloudFlare

Question

I'm trying to download this file from CloudFlare using PlayWright in Python: https://www.historique-meteo.net/site/export.php?ville_id=1067

But impossible to succeed... Could you have any idea to help please ?

Note: there is no click to perform the download, just wait for 5 seconds for JS auto check

Here is my code:

from playwright.sync_api import sync_playwright

def run(playwright):
    browser = playwright.chromium.launch(headless=False)
    context = browser.new_context(accept_downloads=True)

    # Open new page
    page = context.new_page()

    # Go to site
    page.goto("https://www.historique-meteo.net/site/export.php?ville_id=1067")

    # Download
    page.on("download", lambda download: download.save_as(download.suggested_filename))

    context.close()
    browser.close()

with sync_playwright() as playwright:
    run(playwright)

Thanks a lot ! :)

Answer 1

Not guaranteed for success. Change the parameters in function async_cf_retry if it tends to fail. I wrote this code by modifying my heavy modules, so it may not work elegantly.

import re
import asyncio
from playwright.async_api import async_playwright, Error, Page
from cf_clearance import stealth_async
import httpx

# local_client = httpx.AsyncClient(verify=False)


async def async_cf_retry(page: Page, tries=10) -> bool:
    # use tries=-1 for infinite retries
    # excerpted from `from cf_clearance import async_retry`
    success = False
    while tries != 0:
        try:
            title = await page.title()
        except Error:
            tries -= 1
            await asyncio.sleep(1)
        else:
            # print(title)
            if title == 'Please Wait... | Cloudflare':
                await page.close()
                raise NotImplementedError('Encountered recaptcha. Check whether your proxy is an elite proxy.')
            elif title == 'Just a moment...':
                tries -= 1
                await asyncio.sleep(5)
            elif "www." in title:
                await page.reload()
                tries -= 1
                await asyncio.sleep(5)
            else:
                success = True
                break
    return success


async def get_client_with_clearance(proxy: str = None):
    async def get_one_clearance(proxy=proxy, logs=False):
        # proxy = {"server": "socks5://localhost:7890"}
        if type(proxy) is str:
            proxy = {'server': proxy}
        async with async_playwright() as p:
            browser = await p.chromium.launch(headless=False, proxy=proxy,
                                              args=["--window-position=1000,800", "--disable-web-security",
                                                    "--disable-webgl"])
            page = await browser.new_page()#viewport={"width": 0, "height": 0})
            await stealth_async(page)
            
            # Download
            async def on_download(download):
                print('download', download)
                await download.save_as(download.suggested_filename)
            page.on("download", on_download)

            if logs:
                def log_response(intercepted_response):
                    print("a response was received:", intercepted_response.url)
                
                page.on("response", log_response)
            
            await page.goto("https://www.historique-meteo.net/site/export.php?ville_id=1067")
            res = await async_cf_retry(page)
            if res:
                cookies = await page.context.cookies()
                cookies_for_httpx = {cookie['name']: cookie['value'] for cookie in cookies}
                ua = await page.evaluate('() => {return navigator.userAgent}')
                # print(ua)
            else:
                await page.close()
                raise InterruptedError("cf challenge fail")
            await asyncio.sleep(10000)
            return ua, cookies_for_httpx
    
    ua, cookies_for_httpx = await get_one_clearance(logs=True)
    print(cookies_for_httpx)


print(asyncio.get_event_loop().run_until_complete(get_client_with_clearance(
    # proxy='http://localhost:8888'
    # use proxifier on windows as an elite proxy
)))

If download <Download url='https://www.historique-meteo.net/site/export.php?ville_id=1067' suggested_filename='export-aix-les-bains.csv'> is shown in the console, the browser must be downloading your file.
This code may keep trying to get a cloudflare clearance even if the file has been downloaded. Edit your strategies in async_cf_retry if this happens.
Use persisent context offered by playwright to save and reuse your cloudflare clearance. Consider downloading with httpx.