From 6636ac90fa620bbd271a6a0272ed627b83ab0fef Mon Sep 17 00:00:00 2001 From: Mynacol Date: Wed, 21 Jun 2023 16:34:28 +0200 Subject: [PATCH] Use HTTP compression for downloads As long as we use Feed-io < 6, we always use Guzzle (with Feed-io 6, we need to choose a [HTTPlug](https://httplug.io/) library ourselves). Guzzle [supports](https://docs.guzzlephp.org/en/stable/request-options.html#decode-content) transparently decompressing gzip or deflate compressed responses, which is enabled by default. Feed-io does this [by default](https://github.com/alexdebril/feed-io/pull/311/commits/ffef9eaabefcfc66bca3bb598b952d13842f14b5), but as we override the headers, we have to add a fitting `Accept-Encoding` header as well. Previously, my feed collection caused up to 45.9 MB download traffic per cronjob, with this commit it is as low as 23.6 MB. Signed-off-by: Mynacol --- CHANGELOG.md | 1 + lib/Config/FetcherConfig.php | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eda601e5c..fd8c2d744 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is mostly based on [Keep a Changelog](https://keepachangelog.com/en/1 # Unreleased ## [22.x.x] ### Changed +- Support deflate and gzip compression for HTTP response bodies (#2269) ### Fixed diff --git a/lib/Config/FetcherConfig.php b/lib/Config/FetcherConfig.php index ce2e7db8d..797dae49e 100644 --- a/lib/Config/FetcherConfig.php +++ b/lib/Config/FetcherConfig.php @@ -103,7 +103,11 @@ public function getClient(): ClientInterface { $config = [ 'timeout' => $this->client_timeout, - 'headers' => ['User-Agent' => static::DEFAULT_USER_AGENT, 'Accept' => static::DEFAULT_ACCEPT], + 'headers' => [ + 'User-Agent' => static::DEFAULT_USER_AGENT, + 'Accept' => static::DEFAULT_ACCEPT, + 'Accept-Encoding' => 'gzip, deflate', + ], ]; if (!is_null($this->proxy)) {