Add support for gzip and brotli

Automatically sends the Accept-Encoding header on requests.

Not runtime-configurable, only with Cargo features.

Cargo.toml

@@ -22,6 +22,8 @@ json = ["serde", "serde_json"]
 charset = ["encoding_rs"]
 cookies = ["cookie", "cookie_store"]
 socks-proxy = ["socks"]
+gzip = ["flate2"]
+brotli = ["brotli-decompressor"]
 
 [dependencies]
 base64 = "0.13"
@@ -40,6 +42,8 @@ webpki-roots = { version = "0.22", optional = true }
 rustls = { version = "0.20", optional = true }
 rustls-native-certs = { version = "0.6", optional = true }
 native-tls = { version = "0.2", optional = true }
+flate2 = { version = "1.0.22", optional = true }
+brotli-decompressor = { version = "2.3.2", optional = true }
 
 [dev-dependencies]
 serde = { version = "1", features = ["derive"] }

src/request.rs

@@ -87,12 +87,36 @@ impl Request {
         )?)
     }
 
-    fn do_call(self, payload: Payload) -> Result<Response> {
+    /// Add Accept-Encoding header with supported values, unless user has
+    /// already set this header or is requesting a specific byte-range.
+    #[cfg(any(feature = "gzip", feature = "brotli"))]
+    fn add_accept_encoding(&mut self) {
+        let should_add = !self.headers.iter().map(|h| h.name()).any(|name| {
+            name.eq_ignore_ascii_case("accept-encoding") || name.eq_ignore_ascii_case("range")
+        });
+        if should_add {
+            const GZ: bool = cfg!(feature = "gzip");
+            const BR: bool = cfg!(feature = "brotli");
+            const ACCEPT: &str = match (GZ, BR) {
+                (true, true) => "gzip, br",
+                (true, false) => "gzip",
+                (false, true) => "br",
+                (false, false) => "identity", // unreachable due to cfg feature on this fn
+            };
+            self.headers.push(Header::new("accept-encoding", ACCEPT));
+        }
+    }
+
+    #[cfg_attr(not(any(feature = "gzip", feature = "brotli")), allow(unused_mut))]
+    fn do_call(mut self, payload: Payload) -> Result<Response> {
         for h in &self.headers {
             h.validate()?;
         }
         let url = self.parse_url()?;
 
+        #[cfg(any(feature = "gzip", feature = "brotli"))]
+        self.add_accept_encoding();
+
         let deadline = match self.timeout.or(self.agent.config.timeout) {
             None => None,
             Some(timeout) => {

src/response.rs

@@ -18,6 +18,12 @@ use serde::de::DeserializeOwned;
 #[cfg(feature = "charset")]
 use encoding_rs::Encoding;
 
+#[cfg(feature = "gzip")]
+use flate2::read::GzDecoder;
+
+#[cfg(feature = "brotli")]
+use brotli_decompressor::Decompressor;
+
 pub const DEFAULT_CONTENT_TYPE: &str = "text/plain";
 pub const DEFAULT_CHARACTER_SET: &str = "utf-8";
 const INTO_STRING_LIMIT: usize = 10 * 1_024 * 1_024;
@@ -281,6 +287,11 @@ impl Response {
                 .and_then(|l| l.parse::<usize>().ok())
         };
 
+        let compression = self
+            .header("content-encoding")
+            .map(Compression::from_header_value)
+            .flatten();
+
         let stream = self.stream;
         let unit = self.unit;
         if let Some(unit) = &unit {
@@ -292,12 +303,17 @@ impl Response {
         let deadline = unit.as_ref().and_then(|u| u.deadline);
         let stream = DeadlineStream::new(*stream, deadline);
 
-        match (use_chunked, limit_bytes) {
+        let body_reader: Box<dyn Read + Send> = match (use_chunked, limit_bytes) {
             (true, _) => Box::new(PoolReturnRead::new(unit, ChunkDecoder::new(stream))),
             (false, Some(len)) => {
                 Box::new(PoolReturnRead::new(unit, LimitedRead::new(stream, len)))
             }
             (false, None) => Box::new(stream),
+        };
+
+        match compression {
+            None => body_reader,
+            Some(c) => c.wrap_reader(body_reader),
         }
     }
 
@@ -508,6 +524,38 @@ impl Response {
     }
 }
 
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+enum Compression {
+    #[cfg(feature = "brotli")]
+    Brotli,
+    #[cfg(feature = "gzip")]
+    Gzip,
+}
+
+impl Compression {
+    /// Convert a string like "br" to an enum value
+    fn from_header_value(value: &str) -> Option<Compression> {
+        match value {
+            #[cfg(feature = "brotli")]
+            "br" => Some(Compression::Brotli),
+            #[cfg(feature = "gzip")]
+            "gzip" | "x-gzip" => Some(Compression::Gzip),
+            _ => None,
+        }
+    }
+
+    /// Wrap the raw reader with a decompressing reader
+    #[allow(unused_variables)] // when no features enabled, reader is unused (unreachable)
+    fn wrap_reader(self, reader: Box<dyn Read + Send>) -> Box<dyn Read + Send> {
+        match self {
+            #[cfg(feature = "brotli")]
+            Compression::Brotli => Box::new(Decompressor::new(reader, 4096)),
+            #[cfg(feature = "gzip")]
+            Compression::Gzip => Box::new(GzDecoder::new(reader)),
+        }
+    }
+}
+
 /// parse a line like: HTTP/1.1 200 OK\r\n
 fn parse_status_line(line: &str) -> Result<(ResponseStatusIndex, u16), Error> {
     //