diff --git a/Cargo.toml b/Cargo.toml index b820382..76a4929 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,8 @@ json = ["serde", "serde_json"] charset = ["encoding_rs"] cookies = ["cookie", "cookie_store"] socks-proxy = ["socks"] +gzip = ["flate2"] +brotli = ["brotli-decompressor"] [dependencies] base64 = "0.13" @@ -40,6 +42,8 @@ webpki-roots = { version = "0.22", optional = true } rustls = { version = "0.20", optional = true } rustls-native-certs = { version = "0.6", optional = true } native-tls = { version = "0.2", optional = true } +flate2 = { version = "1.0.22", optional = true } +brotli-decompressor = { version = "2.3.2", optional = true } [dev-dependencies] serde = { version = "1", features = ["derive"] } diff --git a/src/request.rs b/src/request.rs index cfb7b16..af0c862 100644 --- a/src/request.rs +++ b/src/request.rs @@ -87,12 +87,36 @@ impl Request { )?) } - fn do_call(self, payload: Payload) -> Result { + /// Add Accept-Encoding header with supported values, unless user has + /// already set this header or is requesting a specific byte-range. + #[cfg(any(feature = "gzip", feature = "brotli"))] + fn add_accept_encoding(&mut self) { + let should_add = !self.headers.iter().map(|h| h.name()).any(|name| { + name.eq_ignore_ascii_case("accept-encoding") || name.eq_ignore_ascii_case("range") + }); + if should_add { + const GZ: bool = cfg!(feature = "gzip"); + const BR: bool = cfg!(feature = "brotli"); + const ACCEPT: &str = match (GZ, BR) { + (true, true) => "gzip, br", + (true, false) => "gzip", + (false, true) => "br", + (false, false) => "identity", // unreachable due to cfg feature on this fn + }; + self.headers.push(Header::new("accept-encoding", ACCEPT)); + } + } + + #[cfg_attr(not(any(feature = "gzip", feature = "brotli")), allow(unused_mut))] + fn do_call(mut self, payload: Payload) -> Result { for h in &self.headers { h.validate()?; } let url = self.parse_url()?; + #[cfg(any(feature = "gzip", feature = "brotli"))] + self.add_accept_encoding(); + let deadline = match self.timeout.or(self.agent.config.timeout) { None => None, Some(timeout) => { diff --git a/src/response.rs b/src/response.rs index e5e0150..becd9ed 100644 --- a/src/response.rs +++ b/src/response.rs @@ -18,6 +18,12 @@ use serde::de::DeserializeOwned; #[cfg(feature = "charset")] use encoding_rs::Encoding; +#[cfg(feature = "gzip")] +use flate2::read::GzDecoder; + +#[cfg(feature = "brotli")] +use brotli_decompressor::Decompressor; + pub const DEFAULT_CONTENT_TYPE: &str = "text/plain"; pub const DEFAULT_CHARACTER_SET: &str = "utf-8"; const INTO_STRING_LIMIT: usize = 10 * 1_024 * 1_024; @@ -281,6 +287,11 @@ impl Response { .and_then(|l| l.parse::().ok()) }; + let compression = self + .header("content-encoding") + .map(Compression::from_header_value) + .flatten(); + let stream = self.stream; let unit = self.unit; if let Some(unit) = &unit { @@ -292,12 +303,17 @@ impl Response { let deadline = unit.as_ref().and_then(|u| u.deadline); let stream = DeadlineStream::new(*stream, deadline); - match (use_chunked, limit_bytes) { + let body_reader: Box = match (use_chunked, limit_bytes) { (true, _) => Box::new(PoolReturnRead::new(unit, ChunkDecoder::new(stream))), (false, Some(len)) => { Box::new(PoolReturnRead::new(unit, LimitedRead::new(stream, len))) } (false, None) => Box::new(stream), + }; + + match compression { + None => body_reader, + Some(c) => c.wrap_reader(body_reader), } } @@ -508,6 +524,38 @@ impl Response { } } +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum Compression { + #[cfg(feature = "brotli")] + Brotli, + #[cfg(feature = "gzip")] + Gzip, +} + +impl Compression { + /// Convert a string like "br" to an enum value + fn from_header_value(value: &str) -> Option { + match value { + #[cfg(feature = "brotli")] + "br" => Some(Compression::Brotli), + #[cfg(feature = "gzip")] + "gzip" | "x-gzip" => Some(Compression::Gzip), + _ => None, + } + } + + /// Wrap the raw reader with a decompressing reader + #[allow(unused_variables)] // when no features enabled, reader is unused (unreachable) + fn wrap_reader(self, reader: Box) -> Box { + match self { + #[cfg(feature = "brotli")] + Compression::Brotli => Box::new(Decompressor::new(reader, 4096)), + #[cfg(feature = "gzip")] + Compression::Gzip => Box::new(GzDecoder::new(reader)), + } + } +} + /// parse a line like: HTTP/1.1 200 OK\r\n fn parse_status_line(line: &str) -> Result<(ResponseStatusIndex, u16), Error> { //