From 26ef48c8899e1a9d1da54ba4ed6b0a41353d603c Mon Sep 17 00:00:00 2001 From: Andy Date: Wed, 30 Jul 2025 00:32:25 +0000 Subject: [PATCH] =?UTF-8?q?fix(download):=20=F0=9F=90=9B=20Skip=20Content-?= =?UTF-8?q?Length=20validation=20for=20compressed=20responses=20in=20curl?= =?UTF-8?q?=5Fimpersonate=20and=20requests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- unshackle/core/downloaders/curl_impersonate.py | 5 +++++ unshackle/core/downloaders/requests.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/unshackle/core/downloaders/curl_impersonate.py b/unshackle/core/downloaders/curl_impersonate.py index f9f0d95..52dab7a 100644 --- a/unshackle/core/downloaders/curl_impersonate.py +++ b/unshackle/core/downloaders/curl_impersonate.py @@ -76,6 +76,11 @@ def download(url: str, save_path: Path, session: Session, **kwargs: Any) -> Gene try: content_length = int(stream.headers.get("Content-Length", "0")) + + # Skip Content-Length validation for compressed responses since + # curl_impersonate automatically decompresses but Content-Length shows compressed size + if stream.headers.get("Content-Encoding", "").lower() in ["gzip", "deflate", "br"]: + content_length = 0 except ValueError: content_length = 0 diff --git a/unshackle/core/downloaders/requests.py b/unshackle/core/downloaders/requests.py index ef57d77..49c1759 100644 --- a/unshackle/core/downloaders/requests.py +++ b/unshackle/core/downloaders/requests.py @@ -90,6 +90,11 @@ def download( if not segmented: try: content_length = int(stream.headers.get("Content-Length", "0")) + + # Skip Content-Length validation for compressed responses since + # requests automatically decompresses but Content-Length shows compressed size + if stream.headers.get("Content-Encoding", "").lower() in ["gzip", "deflate", "br"]: + content_length = 0 except ValueError: content_length = 0