Петр Василий
Reputation: 1
Different responses to the same requests (curl, python aiohttp) on the server and on the local host
I have a parser written in Python (aiohttp, bs4). One of the functions of the parser is to access a file by a link (example: https://modsfire.com/d/Mwv01aESgj73zx7).
import aiohttp
import yarl
import asyncio
from pprint import pprint
MODSFIRE_URL = "https://modsfire.com/"
COOKIES = {
'_ga_JXQKZFEW04': 'GS1.1.1718453797.5.1.1718456834.0.0.0',
'_ga': 'GA1.1.733263822.1718378970',
'old_date': 'eyJpdiI6IlA0NXg3aDV6QVBzazFMVHRibjR3NWc9PSIsInZhbHVlIjoiXC80RWt3ZHNNdEFQMks0SGhjdEltNEE9PSIsIm1hYyI6ImM3Mzg4NzkyZTA5YjI5YjRiZDc2YjdkODIyMDEwNDE4MzBmZWJlYWU0MTc0ZmMxZGQxNTc0MjQyN2NlMGE1YzYifQ%3D%3D',
'fid': 'eyJpdiI6IjFKSjJ1VDVaSHVpdmh4ZWdJcjBiZ2c9PSIsInZhbHVlIjoicTArdExGeDdPRFpua0dVWm1vZ1pRZz09IiwibWFjIjoiNjJjZDZjZDg1NzgxODk1NmJkNmNmMDBiOTg1OWUzMTM0MmU1MjI3ODExZjkzMGRkOTBhMWVmM2RkOTJhMGM3YSJ9',
'XSRF-TOKEN': 'eyJpdiI6ImNNM20wcjhLMnBBMW9sRG5VZm9rK1E9PSIsInZhbHVlIjoiRU9MRUVcLzExN0RiU1d2VXlXY0ExQ0tpbVwveGlDbDdiTFNpbURWSlZEZE4reEM0VlRxc1JWVFYyR3NLOTVwaGIyIiwibWFjIjoiMmUwOTNlYTIyYTkxMDZkNDJiNzY0NDE5ODY4ZDU1NGEwODMwNmIyYjZjZjliYzUxNTc3ZDZjNzA0ZGI5MTQwYiJ9',
'modsfire_session': 'eyJpdiI6Ikxia0dmMVwvMGtSQ3VyWWJBWHRadUZ3PT0iLCJ2YWx1ZSI6IjNmeFBkcmtWZ0I4S3l1WDB6eGhTT2JyTnBlajlcL3pxMGlubWNRR1NvUlZHQlI5eWMxVmpPelNhTzBjYUoyVXBzIiwibWFjIjoiODg5YTE2N2UzZGI1NDRhN2QwNGFlNjk3NDEwMDNkZTY5MThkZjJiZjE5ZDIyZGNlYjNhZWM1NDY2ODg4ZWU4NyJ9',
'referer_domain': 'eyJpdiI6ImdVM2lZYWRQdm1iNEkycXRJNGpRM3c9PSIsInZhbHVlIjoid1NmU0F5b2NiS0JpWk4yR051YVN2QT09IiwibWFjIjoiODExMDllOGQ4ZmU1Y2RjMWVkZTIxNmFhNzhiYzNmMTFkZWNmODVmZDc5NDFlMWJhNDhlMmNmNjE2MWQzZTIwYyJ9',
'cf_clearance': 'UFEt8KWYJuVxadyvHEgHGOcRxIzVrFYvF2h.KHdSeD8-1718456567-1.0.1.1-CmNj9kZt85DTea1QAvCxuIwwb3KzNJ96jRBsoM7r1oOOeMcNeLa3XS4osEElROQq0oely9xLAIj_U05z2sCiZg',
'a644b2a3a3018e5f62e31f82c6db5ae0': 'eyJpdiI6Ijl6eFwvWXp3UkpUWUhYcWVqYzlDb3R3PT0iLCJ2YWx1ZSI6IlI2YW1FZ2JkQmIyZm9wY0FUNkVEdGc9PSIsIm1hYyI6ImY5OGJlN2FkZjljZWYzNTM1ZjM5OGZjMmU1OGY0MWE2NTFiYjYyNTcwNDUzYzg1ZTJlNGNkMjc0ZmZiZjJiMTMifQ%3D%3D',
}
HEADERS = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Accept-Language': 'ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3',
# 'Accept-Encoding': 'gzip, deflate, br, zstd',
'Referer': MODSFIRE_URL,
'Connection': 'keep-alive',
# 'Cookie': '_ga_JXQKZFEW04=GS1.1.1718453797.5.1.1718458248.0.0.0; _ga=GA1.1.733263822.1718378970; cf_clearance=UFEt8KWYJuVxadyvHEgHGOcRxIzVrFYvF2h.KHdSeD8-1718456567-1.0.1.1-CmNj9kZt85DTea1QAvCxuIwwb3KzNJ96jRBsoM7r1oOOeMcNeLa3XS4osEElROQq0oely9xLAIj_U05z2sCiZg',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-site',
'Sec-Fetch-User': '?1',
'Priority': 'u=1',
# Requests doesn't support trailers
# 'TE': 'trailers',
}
async def get_response_file(url):
session = aiohttp.ClientSession(headers=HEADERS, cookies=COOKIES)
response = await session.get(url, allow_redirects=True)
pprint(dict(response.request_info.headers))
print("===============")
pprint(dict(response.headers))
if response.status != 200:
return "Error"
filename = yarl.URL(response.url).name
response_headers = dict(response.headers)
if filename != "d.php":
response_headers["Content-Disposition"] = "attachment; filename=" + filename
response_headers["Content-Type"] = "application/octet-stream"
return response, response_headers
asyncio.run(get_response_file("https://modsfire.com/d/Mwv01aESgj73zx7"))
On my machine everything works - I get a 200 response. But as soon as I run this script in docker, wine or on my server - I get response 403.
On my machine the output is:
{'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3',
'Connection': 'keep-alive',
'Cookie': 'XSRF-TOKEN=eyJpdiI6ImNNM20wcjhLMnBBMW9sRG5VZm9rK1E9PSIsInZhbHVlIjoiRU9MRUVcLzExN0RiU1d2VXlXY0ExQ0tpbVwveGlDbDdiTFNpbURWSlZEZE4reEM0VlRxc1JWVFYyR3NLOTVwaGIyIiwibWFjIjoiMmUwOTNlYTIyYTkxMDZkNDJiNzY0NDE5ODY4ZDU1NGEwODMwNmIyYjZjZjliYzUxNTc3ZDZjNzA0ZGI5MTQwYiJ9; '
'_ga=GA1.1.733263822.1718378970; '
'_ga_JXQKZFEW04=GS1.1.1718453797.5.1.1718456834.0.0.0; '
'a644b2a3a3018e5f62e31f82c6db5ae0=eyJpdiI6Ijl6eFwvWXp3UkpUWUhYcWVqYzlDb3R3PT0iLCJ2YWx1ZSI6IlI2YW1FZ2JkQmIyZm9wY0FUNkVEdGc9PSIsIm1hYyI6ImY5OGJlN2FkZjljZWYzNTM1ZjM5OGZjMmU1OGY0MWE2NTFiYjYyNTcwNDUzYzg1ZTJlNGNkMjc0ZmZiZjJiMTMifQ%3D%3D; '
'cf_clearance=UFEt8KWYJuVxadyvHEgHGOcRxIzVrFYvF2h.KHdSeD8-1718456567-1.0.1.1-CmNj9kZt85DTea1QAvCxuIwwb3KzNJ96jRBsoM7r1oOOeMcNeLa3XS4osEElROQq0oely9xLAIj_U05z2sCiZg; '
'fid=eyJpdiI6IjFKSjJ1VDVaSHVpdmh4ZWdJcjBiZ2c9PSIsInZhbHVlIjoicTArdExGeDdPRFpua0dVWm1vZ1pRZz09IiwibWFjIjoiNjJjZDZjZDg1NzgxODk1NmJkNmNmMDBiOTg1OWUzMTM0MmU1MjI3ODExZjkzMGRkOTBhMWVmM2RkOTJhMGM3YSJ9; '
'modsfire_session=eyJpdiI6Ikxia0dmMVwvMGtSQ3VyWWJBWHRadUZ3PT0iLCJ2YWx1ZSI6IjNmeFBkcmtWZ0I4S3l1WDB6eGhTT2JyTnBlajlcL3pxMGlubWNRR1NvUlZHQlI5eWMxVmpPelNhTzBjYUoyVXBzIiwibWFjIjoiODg5YTE2N2UzZGI1NDRhN2QwNGFlNjk3NDEwMDNkZTY5MThkZjJiZjE5ZDIyZGNlYjNhZWM1NDY2ODg4ZWU4NyJ9; '
'old_date=eyJpdiI6IlA0NXg3aDV6QVBzazFMVHRibjR3NWc9PSIsInZhbHVlIjoiXC80RWt3ZHNNdEFQMks0SGhjdEltNEE9PSIsIm1hYyI6ImM3Mzg4NzkyZTA5YjI5YjRiZDc2YjdkODIyMDEwNDE4MzBmZWJlYWU0MTc0ZmMxZGQxNTc0MjQyN2NlMGE1YzYifQ%3D%3D; '
'referer_domain=eyJpdiI6ImdVM2lZYWRQdm1iNEkycXRJNGpRM3c9PSIsInZhbHVlIjoid1NmU0F5b2NiS0JpWk4yR051YVN2QT09IiwibWFjIjoiODExMDllOGQ4ZmU1Y2RjMWVkZTIxNmFhNzhiYzNmMTFkZWNmODVmZDc5NDFlMWJhNDhlMmNmNjE2MWQzZTIwYyJ9',
'Host': 'r3-kjxyo-modsfire.b6fbbdf4a435e192350b1c62f9bdcba7.r2.cloudflarestorage.com',
'Priority': 'u=1',
'Referer': 'https://modsfire.com/',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-site',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 '
'Firefox/127.0'}
===============
{'Accept-Ranges': 'bytes',
'CF-RAY': '8a6b98178d2d9d71-DME',
'Connection': 'keep-alive',
'Content-Length': '3982831',
'Content-Type': 'application/zip',
'Date': 'Sun, 21 Jul 2024 13:38:55 GMT',
'Etag': '"8c89e580f490fc3d9ff50098b7967cc1-1"',
'Last-Modified': 'Sat, 20 Jul 2024 08:42:18 GMT',
'Server': 'cloudflare',
'Vary': 'Accept-Encoding',
'x-amz-mp-parts-count': '1'}
On the server, in the docker and in the wine:
# Same
===============
{'Accept-CH': 'Sec-CH-UA-Bitness, Sec-CH-UA-Arch, Sec-CH-UA-Full-Version, '
'Sec-CH-UA-Mobile, Sec-CH-UA-Model, Sec-CH-UA-Platform-Version, '
'Sec-CH-UA-Full-Version-List, Sec-CH-UA-Platform, Sec-CH-UA, '
'UA-Bitness, UA-Arch, UA-Full-Version, UA-Mobile, UA-Model, '
'UA-Platform-Version, UA-Platform, UA',
'CF-RAY': '8a6b99d1ec1eabe6-CPH',
'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, '
'post-check=0, pre-check=0',
'Connection': 'close',
'Content-Encoding': 'gzip',
'Content-Type': 'text/html; charset=UTF-8',
'Critical-CH': 'Sec-CH-UA-Bitness, Sec-CH-UA-Arch, Sec-CH-UA-Full-Version, '
'Sec-CH-UA-Mobile, Sec-CH-UA-Model, '
'Sec-CH-UA-Platform-Version, Sec-CH-UA-Full-Version-List, '
'Sec-CH-UA-Platform, Sec-CH-UA, UA-Bitness, UA-Arch, '
'UA-Full-Version, UA-Mobile, UA-Model, UA-Platform-Version, '
'UA-Platform, UA',
'Cross-Origin-Embedder-Policy': 'require-corp',
'Cross-Origin-Opener-Policy': 'same-origin',
'Cross-Origin-Resource-Policy': 'same-origin',
'Date': 'Sun, 21 Jul 2024 13:40:06 GMT',
'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT',
'NEL': '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}',
'Origin-Agent-Cluster': '?1',
'Permissions-Policy': 'accelerometer=(),autoplay=(),browsing-topics=(),camera=(),clipboard-r
ead=(),clipboard-write=(),geolocation=(),gyroscope=(),hid=(),interest-cohort=(),magnetometer=
(),microphone=(),payment=(),publickey-credentials-get=(),screen-wake-lock=(),serial=(),sync-x
hr=(),usb=()',
'Referrer-Policy': 'same-origin',
'Report-To': '{"endpoints":[{"url":"https:\\/\\/a.nel.cloudflare.com\\/report\\/v4?s=BCuZckp
VeM5bhyVEA5%2FVN5jQZt%2BlyC85kdRzK0gYtoIWSBiTWJNniVedtDnL6aUGFb0lWXGCbpfw3Qywn213tqKdifufZzgb
Z8lQ4UukRLcuX%2BZI4me4bg1Xvhaedw%3D%3D"}],"group":"cf-nel","max_age":604800}',
'Server': 'cloudflare',
'Transfer-Encoding': 'chunked',
'Vary': 'Accept-Encoding',
'X-Content-Options': 'nosniff',
'X-Frame-Options': 'SAMEORIGIN',
'cf-chl-out': '3enpa3K5aSwfgNUzBywARRQ2ul4EUallXhghfa7HgoaNg/qNRItjRdQniZy/yLWF1RC4Ijl8j6qXQ
oKg/78SoT6Q31iobZKau7Yv0T6d8p7j1McT6tBUCvZp3uUAYWl06ktHE67A2m//cyNFHnbQTa93JyyGZ3d9sjFZ4UH0rA
E=$My/VujbV2w+XBN0G4CEg3w==',
'cf-mitigated': 'challenge'}
My machine:
OS: Arch Linux x86_64
WM: bspwm
Kernel: 6.9.10-zen1-1-zen
Python 3.12.4
aiohttp==3.9.5
curl 8.8.0 (x86_64-pc-linux-gnu) libcurl/8.8.0 OpenSSL/3.3.1 zlib/1.3.1 brotli/1.1.0 zstd/1.5.6 libidn2/2.3.7 libpsl/0.21.5 libssh2/1.11.0 nghttp2/1.62.1 nghttp3/1.4.0
Release-Date: 2024-05-22
Docker version 27.0.3, build 7d4bcd863a
wine-9.13
Characteristics of my server: enter image description here
- I was able to run the script in wine, using my computer as a proxy (using mitmproxy).
- I've also tried using public proxies - didn't help
- The curl request to url works on my machine, and on my server (the answers the same), but in docker it doesn't: Curl command:
curl '<url>' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' -H 'Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3' -H 'Accept-Encoding: gzip, deflate, br, zstd' -H 'Referer: https://modsfire.com/' -L -v
Docker command:
docker run --rm curlimages/curl 'https://modsfire.com/d/Mwv01aESgj73zx7' -H 'User-A
gent: Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' -H 'Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3' -H 'Accept-Encoding: gzip, deflate, br, zstd' -H 'Referer: https://modsfire.com/' -L -v
Response in docker:
* IPv6: 2a06:98c1:3122:e000::, 2a06:98c1:3123:e000::
* IPv4: 188.114.98.224, 188.114.99.224
* Trying 188.114.98.224:443...
* Connected to modsfire.com (188.114.98.224) port 443
* ALPN: curl offers h2,http/1.1
} [5 bytes data]
* TLSv1.3 (OUT), TLS handshake, Client hello (1):
} [512 bytes data]
* CAfile: /cacert.pem
* CApath: /etc/ssl/certs
{ [5 bytes data]
* TLSv1.3 (IN), TLS handshake, Server hello (2):
{ [122 bytes data]
* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
{ [19 bytes data]
* TLSv1.3 (IN), TLS handshake, Certificate (11):
{ [4154 bytes data]
* TLSv1.3 (IN), TLS handshake, CERT verify (15):
{ [79 bytes data]
* TLSv1.3 (IN), TLS handshake, Finished (20):
{ [52 bytes data]
* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
} [1 bytes data]
* TLSv1.3 (OUT), TLS handshake, Finished (20):
} [52 bytes data]
* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / X25519 / id-ecPublicKey
* ALPN: server accepted h2
* Server certificate:
* subject: CN=modsfire.com
* start date: May 29 03:12:54 2024 GMT
* expire date: Aug 27 03:12:53 2024 GMT
* subjectAltName: host "modsfire.com" matched cert's "modsfire.com"
* issuer: C=US; O=Let's Encrypt; CN=E1
* SSL certificate verify ok.
* Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384
* Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using ecdsa-with-SHA384
* Certificate level 2: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using ecdsa-with-SHA384
} [5 bytes data]
* using HTTP/2
* [HTTP/2] [1] OPENED stream for https://modsfire.com/d/Mwv01aESgj73zx7
* [HTTP/2] [1] [:method: GET]
* [HTTP/2] [1] [:scheme: https]
* [HTTP/2] [1] [:authority: modsfire.com]
* [HTTP/2] [1] [:path: /d/Mwv01aESgj73zx7]
* [HTTP/2] [1] [user-agent: Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0]
* [HTTP/2] [1] [accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8]
* [HTTP/2] [1] [accept-language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3]
* [HTTP/2] [1] [accept-encoding: gzip, deflate, br, zstd]
* [HTTP/2] [1] [referer: https://modsfire.com/]
} [5 bytes data]
> GET /d/Mwv01aESgj73zx7 HTTP/2
> Host: modsfire.com
> User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0
> Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8
> Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3
> Accept-Encoding: gzip, deflate, br, zstd
> Referer: https://modsfire.com/
>
* Request completely sent off
{ [5 bytes data]
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
{ [230 bytes data]
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
{ [230 bytes data]
* old SSL session ID is stale, removing
{ [5 bytes data]
< HTTP/2 403
< date: Sun, 21 Jul 2024 13:38:00 GMT
< content-type: text/html; charset=UTF-8
< accept-ch: Sec-CH-UA-Bitness, Sec-CH-UA-Arch, Sec-CH-UA-Full-Version, Sec-CH-UA-Mobile, Sec-CH-UA-Model, Sec-CH-UA-Platform-Version, Sec-CH-UA-Full-Version-List, Sec-CH-UA-Platform, Sec-CH-UA, UA-Bitness, UA-Arch, UA-Full-Version, UA-Mobile, UA-Model, UA-Platform-Version, UA-Platform, UA
< critical-ch: Sec-CH-UA-Bitness, Sec-CH-UA-Arch, Sec-CH-UA-Full-Version, Sec-CH-UA-Mobile, Sec-CH-UA-Model, Sec-CH-UA-Platform-Version, Sec-CH-UA-Full-Version-List, Sec-CH-UA-Platform, Sec-CH-UA, UA-Bitness, UA-Arch, UA-Full-Version, UA-Mobile, UA-Model, UA-Platform-Version, UA-Platform, UA
< cross-origin-embedder-policy: require-corp
< cross-origin-opener-policy: same-origin
< cross-origin-resource-policy: same-origin
< origin-agent-cluster: ?1
< permissions-policy: accelerometer=(),autoplay=(),browsing-topics=(),camera=(),clipboard-read=(),clipboard-write=(),geolocation=(),gyroscope=(),hid=(),interest-cohort=(),magnetometer=(),microphone=(),payment=(),publickey-credentials-get=(),screen-wake-lock=(),serial=(),sync-xhr=(),usb=()
< referrer-policy: same-origin
< x-content-options: nosniff
< x-frame-options: SAMEORIGIN
< cf-mitigated: challenge
< cf-chl-out: KHfPV3kjAY1jb1FNEnUvyKvyMVLML41D/a6xL2gye+3Cu93GAZKtKzdRjuCtLK+N5dIsaUuLj4W/YtllqUwqxApBWZvh3mNcu+h1r0+7SE9oymrfiLv43SJpqzBn0TBL1y3Vr02nNBjFRvyaY8ELJA==$y8fX1XwUkq1ES/25ev0iRQ==
< cache-control: private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0
< expires: Thu, 01 Jan 1970 00:00:01 GMT
{ [5 bytes data]
< report-to: {"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=nyb5KfWPFBTwPJQjkO64edjYPXs2V3X1SWQMMU8Al4M91hTEz%2Fo6L098YqRfGjIqator%2FimNVjm9E%2BoYnylQHhZy2ambz4mL2jTDHP283Ux%2F9V6V5%2BvDOH672349CA%3D%3D"}],"group":"cf-nel","max_age":604800}
< nel: {"success_fraction":0,"report_to":"cf-nel","max_age":604800}
< vary: Accept-Encoding
< server: cloudflare
< cf-ray: 8a6b96c18ca19d8b-DME
< content-encoding: br
<
{ [5 bytes data]
Why does the curl request work but the python script on aiohttp does not (on my server)?
Upvotes: 0
Views: 87
Answers (0)
Related Questions
- What is the difference between @staticmethod and @classmethod in Python?
- Catch and print full Python exception traceback without halting/exiting the program
- What are the differences between the urllib, urllib2, urllib3 and requests module?
- What is the naming convention in Python for variables and functions?
- How can I see the request headers made by curl when sending a request to the server?
- Missing custom headers in Google Cloud Storage
- HTTP request using cURL works but doesn't work in Python requests
- What is the difference between old style and new style classes in Python?
- Google suggest query using curl