lib/ureq
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for alternate TLs implementations.

Browse Source
This commit is contained in:
Jacob Hoffman-Andrews
2021-10-04 22:47:00 -07:00
committed by Martin Algesten
parent 1c1dfaa691
commit 56276c3742
17 changed files with 527 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/test.yml vendored
View File

@@ -39,7 +39,7 @@ jobs:
with:
command: doc
# Keep in sync with Cargo.toml's [package.metadata.docs.rs]
args: --no-default-features --no-deps --features "tls json charset cookies socks-proxy"
args: --no-default-features --no-deps --features "tls native-tls json charset cookies socks-proxy"
build_and_test:
name: Test
runs-on: ubuntu-latest
@@ -47,7 +47,8 @@ jobs:
matrix:
tls:
- ""
- tls
- "tls"
- "native-tls"
feature:
- ""
- json
@@ -69,4 +70,4 @@ jobs:
uses: actions-rs/cargo@v1
with:
command: test
args: --no-default-features --features "${{ matrix.tls }} ${{ matrix.feature }}"
args: --no-default-features --features "${{ matrix.tls }}" "${{ matrix.feature }}"

17
Cargo.toml
View File

@@ -12,14 +12,14 @@ edition = "2018"
[package.metadata.docs.rs]
# Keep in sync with .github/workflows/test.yml
features = [ "tls", "json", "charset", "cookies", "socks-proxy" ]
features = ["tls", "native-tls", "json", "charset", "cookies", "socks-proxy"]
[features]
default = ["tls"]
tls = ["webpki", "webpki-roots", "rustls"]
native-certs = ["rustls-native-certs"]
json = ["serde", "serde_json"]
charset = ["encoding_rs"]
tls = ["rustls", "webpki", "webpki-roots"]
native-certs = ["rustls-native-certs"]
cookies = ["cookie", "cookie_store"]
socks-proxy = ["socks"]
@@ -30,15 +30,16 @@ cookie = { version = "0.15", default-features = false, optional = true}
once_cell = "1"
url = "2"
socks = { version = "0.3", optional = true }
rustls = { version = "0.20.1", optional = true }
webpki = { version = "0.22", optional = true }
webpki-roots = { version = "0.22", optional = true }
rustls-native-certs = { version = "0.6", optional = true }
serde = { version = "1", optional = true }
serde_json = { version = "1", optional = true }
encoding_rs = { version = "0.8", optional = true }
cookie_store = { version = "0.15", optional = true, default-features = false, features = ["preserve_order"] }
log = "0.4"
webpki = { version = "0.22", optional = true }
webpki-roots = { version = "0.22", optional = true }
rustls = { version = "0.20", optional = true }
rustls-native-certs = { version = "0.6", optional = true }
native-tls = { version = "0.2", optional = true }
[dev-dependencies]
serde = { version = "1", features = ["derive"] }
@@ -51,4 +52,4 @@ name = "smoke-test"
[[example]]
name = "cureq"
required-features = ["charset", "cookies", "socks-proxy"]
required-features = ["charset", "cookies", "socks-proxy", "native-tls"]

42
README.md
View File

@@ -12,7 +12,7 @@ HTTPS, and charset decoding.
Ureq is in pure Rust for safety and ease of understanding. It avoids using
`unsafe` directly. It [uses blocking I/O][blocking] instead of async I/O, because that keeps
the API simple and and keeps dependencies to a minimum. For TLS, ureq uses
[rustls].
[rustls or native-tls](#tls).
Version 2.0.0 was released recently and changed some APIs. See the [changelog] for details.
@@ -101,12 +101,18 @@ You can control them when including ureq as a dependency.
`ureq = { version = "*", features = ["json", "charset"] }`
* `tls` enables https. This is enabled by default.
* `native-certs` makes the default TLS implementation use the OS' trust store (see TLS doc below).
* `cookies` enables cookies.
* `json` enables [Response::into_json()] and [Request::send_json()] via serde_json.
* `charset` enables interpreting the charset part of the Content-Type header
(e.g. `Content-Type: text/plain; charset=iso-8859-1`). Without this, the
library defaults to Rust's built in `utf-8`.
* `socks-proxy` enables proxy config using the `socks4://`, `socks4a://`, `socks5://` and `socks://` (equal to `socks5://`) prefix.
* `native-tls` enables an adapter so you can pass a `native_tls::TlsConnector` instance
to `AgentBuilder::tls_connector`. Due to the risk of diamond dependencies accidentally switching on an unwanted
TLS implementation, `native-tls` is never picked up as a default or used by the crate level
convenience calls (`ureq::get` etc) – it must be configured on the agent. The `native-certs` feature
does nothing for `native-tls`.
## Plain requests
@@ -211,6 +217,40 @@ fn proxy_example_2() -> std::result::Result<(), ureq::Error> {
}
```
## HTTPS / TLS / SSL
On platforms that support rustls, ureq uses rustls. On other platforms, native-tls can
be manually configured using [`AgentBuilder::tls_connector`].
You might want to use native-tls if you need to interoperate with servers that
only support less-secure TLS configurations (rustls doesn't support TLS 1.0 and 1.1, for
instance). You might also want to use it if you need to validate certificates for IP addresses,
which are not currently supported in rustls.
Here's an example of constructing an Agent that uses native-tls. It requires the
"native-tls" feature to be enabled.
```rust
use std::sync::Arc;
use ureq::Agent;
let agent = ureq::AgentBuilder::new()
.tls_connector(Arc::new(native_tls::TlsConnector::new().unwrap()))
.build();
```
### Trusted Roots
When you use rustls (`tls` feature), ureq defaults to trusting
[webpki-roots](https://docs.rs/webpki-roots/), a
copy of the Mozilla Root program that is bundled into your program (and so won't update if your
program isn't updated). You can alternately configure
[rustls-native-certs](https://docs.rs/rustls-native-certs/) which extracts the roots from your
OS' trust store. That means it will update when your OS is updated, and also that it will
include locally installed roots.
When you use `native-tls`, ureq will use your OS' certificate verifier and root store.
## Blocking I/O for simplicity
Ureq uses blocking I/O rather than Rust's newer [asynchronous (async) I/O][async]. Async I/O

31
examples/cureq/main.rs
View File

@@ -3,13 +3,14 @@ use std::fmt;
use std::io;
use std::thread;
use std::time::Duration;
use std::time::SystemTime;
use std::{env, sync::Arc};
use rustls::{
Certificate, ClientConfig, RootCertStore, ServerCertVerified, ServerCertVerifier, TLSError,
};
use rustls::client::ServerCertVerified;
use rustls::client::ServerCertVerifier;
use rustls::ServerName;
use rustls::{Certificate, ClientConfig};
use ureq;
use webpki::DNSNameRef;
#[derive(Debug)]
struct StringError(String);
@@ -100,11 +101,13 @@ struct AcceptAll {}
impl ServerCertVerifier for AcceptAll {
fn verify_server_cert(
&self,
_roots: &RootCertStore,
_presented_certs: &[Certificate],
_dns_name: DNSNameRef<'_>,
_end_entity: &Certificate,
_intermediates: &[Certificate],
_server_name: &ServerName,
_scts: &mut dyn Iterator<Item = &[u8]>,
_ocsp_response: &[u8],
) -> Result<ServerCertVerified, TLSError> {
_now: SystemTime,
) -> Result<ServerCertVerified, rustls::Error> {
Ok(ServerCertVerified::assertion())
}
}
@@ -132,6 +135,7 @@ fn main2() -> Result<(), Error> {
-k Ignore certificate errors
-m <time> Max time for the entire request
-ct <time> Connection timeout
--native-tls Use native-tls
Fetch url and copy it to stdout.
"##,
@@ -160,12 +164,15 @@ Fetch url and copy it to stdout.
wait = Duration::from_secs(wait_seconds);
}
"-k" => {
let mut client_config = ClientConfig::new();
client_config
.dangerous()
.set_certificate_verifier(Arc::new(AcceptAll {}));
let client_config = ClientConfig::builder()
.with_safe_defaults()
.with_custom_certificate_verifier(Arc::new(AcceptAll {}))
.with_no_client_auth();
builder = builder.tls_config(Arc::new(client_config));
}
"--native-tls" => {
builder = builder.tls_connector(Arc::new(native_tls::TlsConnector::new().unwrap()));
}
"-m" => {
let t: f32 = args
.next()

60
src/agent.rs
View File

@@ -1,3 +1,4 @@
use std::fmt;
use std::sync::Arc;
use url::Url;
@@ -6,6 +7,7 @@ use crate::pool::ConnectionPool;
use crate::proxy::Proxy;
use crate::request::Request;
use crate::resolve::{ArcResolver, StdResolver};
use crate::stream::TlsConnector;
use std::time::Duration;
#[cfg(feature = "cookies")]
@@ -28,7 +30,7 @@ pub struct AgentBuilder {
}
/// Config as built by AgentBuilder and then static for the lifetime of the Agent.
#[derive(Debug, Clone)]
#[derive(Clone)]
pub(crate) struct AgentConfig {
pub proxy: Option<Proxy>,
pub timeout_connect: Option<Duration>,
@@ -37,8 +39,13 @@ pub(crate) struct AgentConfig {
pub timeout: Option<Duration>,
pub redirects: u32,
pub user_agent: String,
#[cfg(feature = "tls")]
pub tls_config: Option<TLSClientConfig>,
pub tls_config: Arc<dyn TlsConnector>,
}
impl fmt::Debug for AgentConfig {
fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result {
todo!()
}
}
/// Agents keep state between requests.
@@ -215,8 +222,7 @@ impl AgentBuilder {
timeout: None,
redirects: 5,
user_agent: format!("ureq/{}", env!("CARGO_PKG_VERSION")),
#[cfg(feature = "tls")]
tls_config: None,
tls_config: crate::default_tls_config(),
},
max_idle_connections: DEFAULT_MAX_IDLE_CONNECTIONS,
max_idle_connections_per_host: DEFAULT_MAX_IDLE_CONNECTIONS_PER_HOST,
@@ -472,9 +478,10 @@ impl AgentBuilder {
self
}
/// Set the TLS client config to use for the connection. See [`ClientConfig`](https://docs.rs/rustls/latest/rustls/struct.ClientConfig.html).
/// Configure TLS options for rustls to use when making HTTPS connections from this Agent.
///
/// This overrides any previous call to tls_config or tls_connector.
///
/// Example:
/// ```
/// # fn main() -> Result<(), ureq::Error> {
/// # ureq::is_test(true);
@@ -497,10 +504,34 @@ impl AgentBuilder {
/// .build();
/// # Ok(())
/// # }
/// ```
#[cfg(feature = "tls")]
pub fn tls_config(mut self, tls_config: Arc<rustls::ClientConfig>) -> Self {
self.config.tls_config = Some(TLSClientConfig(tls_config));
self.config.tls_config = Arc::new(tls_config);
self
}
/// Configure TLS options for a backend other than rustls. The parameter can be a
/// any type which implements the [HttpsConnector] trait. If you enable the native-tls
/// feature, we provide `impl HttpsConnector for native_tls::TlsConnector` so you can pass
/// [`Arc<native_tls::TlsConnector>`](https://docs.rs/native-tls/0.2.7/native_tls/struct.TlsConnector.html).
///
/// This overrides any previous call to tls_config or tls_connector.
///
/// ```
/// # fn main() -> Result<(), ureq::Error> {
/// # ureq::is_test(true);
/// use std::sync::Arc;
/// # #[cfg(feature = "native-tls")]
/// let tls_connector = Arc::new(native_tls::TlsConnector::new().unwrap());
/// # #[cfg(feature = "native-tls")]
/// let agent = ureq::builder()
/// .tls_connector(tls_connector.clone())
/// .build();
/// # Ok(())
/// # }
/// ```
pub fn tls_connector<T: TlsConnector + 'static>(mut self, tls_config: Arc<T>) -> Self {
self.config.tls_config = tls_config;
self
}
@@ -537,17 +568,6 @@ impl AgentBuilder {
}
}
#[cfg(feature = "tls")]
#[derive(Clone)]
pub(crate) struct TLSClientConfig(pub(crate) Arc<rustls::ClientConfig>);
#[cfg(feature = "tls")]
impl std::fmt::Debug for TLSClientConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TLSClientConfig").finish()
}
}
#[cfg(test)]
mod tests {
use super::*;

2
src/header.rs
View File

@@ -64,7 +64,7 @@ impl fmt::Display for HeaderLine {
#[derive(Clone, PartialEq)]
/// Wrapper type for a header field.
/// https://tools.ietf.org/html/rfc7230#section-3.2
/// <https://tools.ietf.org/html/rfc7230#section-3.2>
pub struct Header {
// Line contains the unmodified bytes of single header field.
// It does not contain the final CRLF.

110
src/lib.rs
View File

@@ -16,8 +16,8 @@
//!
//! Ureq is in pure Rust for safety and ease of understanding. It avoids using
//! `unsafe` directly. It [uses blocking I/O][blocking] instead of async I/O, because that keeps
//! the API simple and keeps dependencies to a minimum. For TLS, ureq uses
//! [rustls].
//! the API simple and and keeps dependencies to a minimum. For TLS, ureq uses
//! [rustls or native-tls](#tls).
//!
//! Version 2.0.0 was released recently and changed some APIs. See the [changelog] for details.
//!
@@ -121,12 +121,18 @@
//! `ureq = { version = "*", features = ["json", "charset"] }`
//!
//! * `tls` enables https. This is enabled by default.
//! * `native-certs` makes the default TLS implementation use the OS' trust store (see TLS doc below).
//! * `cookies` enables cookies.
//! * `json` enables [Response::into_json()] and [Request::send_json()] via serde_json.
//! * `charset` enables interpreting the charset part of the Content-Type header
//! (e.g. `Content-Type: text/plain; charset=iso-8859-1`). Without this, the
//! library defaults to Rust's built in `utf-8`.
//! * `socks-proxy` enables proxy config using the `socks4://`, `socks4a://`, `socks5://` and `socks://` (equal to `socks5://`) prefix.
//! * `native-tls` enables an adapter so you can pass a `native_tls::TlsConnector` instance
//! to `AgentBuilder::tls_connector`. Due to the risk of diamond dependencies accidentally switching on an unwanted
//! TLS implementation, `native-tls` is never picked up as a default or used by the crate level
//! convenience calls (`ureq::get` etc) – it must be configured on the agent. The `native-certs` feature
//! does nothing for `native-tls`.
//!
//! # Plain requests
//!
@@ -234,6 +240,46 @@
//! # fn main() {}
//! ```
//!
//! # HTTPS / TLS / SSL
//!
//! On platforms that support rustls, ureq uses rustls. On other platforms, native-tls can
//! be manually configured using [`AgentBuilder::tls_connector`].
//!
//! You might want to use native-tls if you need to interoperate with servers that
//! only support less-secure TLS configurations (rustls doesn't support TLS 1.0 and 1.1, for
//! instance). You might also want to use it if you need to validate certificates for IP addresses,
//! which are not currently supported in rustls.
//!
//! Here's an example of constructing an Agent that uses native-tls. It requires the
//! "native-tls" feature to be enabled.
//!
//! ```no_run
//! # #[cfg(feature = "native-tls")]
//! # fn build() -> std::result::Result<(), ureq::Error> {
//! # ureq::is_test(true);
//! use std::sync::Arc;
//! use ureq::Agent;
//!
//! let agent = ureq::AgentBuilder::new()
//! .tls_connector(Arc::new(native_tls::TlsConnector::new().unwrap()))
//! .build();
//! # Ok(())
//! # }
//! # fn main() {}
//! ```
//!
//! ## Trusted Roots
//!
//! When you use rustls (`tls` feature), ureq defaults to trusting
//! [webpki-roots](https://docs.rs/webpki-roots/), a
//! copy of the Mozilla Root program that is bundled into your program (and so won't update if your
//! program isn't updated). You can alternately configure
//! [rustls-native-certs](https://docs.rs/rustls-native-certs/) which extracts the roots from your
//! OS' trust store. That means it will update when your OS is updated, and also that it will
//! include locally installed roots.
//!
//! When you use `native-tls`, ureq will use your OS' certificate verifier and root store.
//!
//! # Blocking I/O for simplicity
//!
//! Ureq uses blocking I/O rather than Rust's newer [asynchronous (async) I/O][async]. Async I/O
@@ -287,6 +333,46 @@ mod response;
mod stream;
mod unit;
// rustls is our default tls engine. If the feature is on, it will be
// used for the shortcut calls the top of the crate (`ureq::get` etc).
#[cfg(feature = "tls")]
mod rtls;
// native-tls is a feature that must be configured via the AgentBuilder.
// it is never picked up as a default (and never used by `ureq::get` etc).
#[cfg(feature = "native-tls")]
mod ntls;
// If we have rustls compiled, that is the default.
#[cfg(feature = "tls")]
pub(crate) fn default_tls_config() -> std::sync::Arc<dyn TlsConnector> {
rtls::default_tls_config()
}
// Without rustls compiled, we just fail on https when using the shortcut
// calls at the top of the crate (`ureq::get` etc).
#[cfg(not(feature = "tls"))]
pub(crate) fn default_tls_config() -> std::sync::Arc<dyn TlsConnector> {
use crate::stream::HttpsStream;
use std::net::TcpStream;
use std::sync::Arc;
struct NoTlsConfig;
impl TlsConnector for NoTlsConfig {
fn connect(
&self,
_dns_name: &str,
_tcp_stream: TcpStream,
) -> Result<Box<dyn HttpsStream>, crate::error::Error> {
Err(ErrorKind::UnknownScheme
.msg("cannot make HTTPS request because no TLS backend is configured"))
}
}
Arc::new(NoTlsConfig)
}
#[cfg(feature = "cookies")]
mod cookies;
@@ -307,6 +393,7 @@ pub use crate::proxy::Proxy;
pub use crate::request::{Request, RequestUrl};
pub use crate::resolve::Resolver;
pub use crate::response::Response;
pub use crate::stream::TlsConnector;
// re-export
#[cfg(feature = "cookies")]
@@ -439,7 +526,7 @@ mod tests {
#[test]
#[cfg(feature = "tls")]
fn connect_https_google() {
fn connect_https_google_rustls() {
let agent = Agent::new();
let resp = agent.get("https://www.google.com/").call().unwrap();
@@ -451,7 +538,22 @@ mod tests {
}
#[test]
#[cfg(feature = "tls")]
#[cfg(feature = "native-tls")]
fn connect_https_google_native_tls() {
use std::sync::Arc;
let tls_config = native_tls::TlsConnector::new().unwrap();
let agent = builder().tls_connector(Arc::new(tls_config)).build();
let resp = agent.get("https://www.google.com/").call().unwrap();
assert_eq!(
"text/html; charset=ISO-8859-1",
resp.header("content-type").unwrap()
);
assert_eq!("text/html", resp.content_type());
}
#[test]
fn connect_https_invalid_name() {
let result = get("https://example.com{REQUEST_URI}/").call();
let e = ErrorKind::Dns;

30
src/ntls.rs Normal file
View File

@@ -0,0 +1,30 @@
use crate::error::Error;
use crate::error::ErrorKind;
use crate::stream::{HttpsStream, TlsConnector};
use std::net::TcpStream;
use std::sync::Arc;
#[allow(dead_code)]
pub(crate) fn default_tls_config() -> std::sync::Arc<dyn TlsConnector> {
Arc::new(native_tls::TlsConnector::new().unwrap())
}
impl TlsConnector for native_tls::TlsConnector {
fn connect(
&self,
dns_name: &str,
tcp_stream: TcpStream,
) -> Result<Box<dyn HttpsStream>, Error> {
let stream = native_tls::TlsConnector::connect(self, dns_name, tcp_stream)
.map_err(|e| ErrorKind::Dns.new().src(e))?;
Ok(Box::new(stream))
}
}
#[cfg(feature = "native-tls")]
impl HttpsStream for native_tls::TlsStream<TcpStream> {
fn socket(&self) -> Option<&TcpStream> {
Some(self.get_ref())
}
}

4
src/response.rs
View File

@@ -491,8 +491,8 @@ impl Response {
}
#[cfg(test)]
pub fn to_write_vec(self) -> Vec<u8> {
self.stream.to_write_vec()
pub fn as_write_vec(&self) -> &[u8] {
self.stream.as_write_vec()
}
#[cfg(test)]

121
src/rtls.rs Normal file
View File

@@ -0,0 +1,121 @@
use std::convert::TryFrom;
use std::io::{self, Read, Write};
use std::net::TcpStream;
use std::sync::Arc;
use once_cell::sync::Lazy;
use crate::ErrorKind;
use crate::{
stream::{HttpsStream, TlsConnector},
Error,
};
#[allow(deprecated)]
fn is_close_notify(e: &std::io::Error) -> bool {
if e.kind() != io::ErrorKind::ConnectionAborted {
return false;
}
if let Some(msg) = e.get_ref() {
// :(
return msg.description().contains("CloseNotify");
}
false
}
struct RustlsStream(rustls::StreamOwned<rustls::ClientConnection, TcpStream>);
impl HttpsStream for RustlsStream {
fn socket(&self) -> Option<&TcpStream> {
Some(self.0.get_ref())
}
}
// TODO: After upgrading to rustls 0.20 or higher, we can remove these Read
// and Write impls, leaving only `impl TlsStream for rustls::StreamOwned...`.
// Currently we need to implement Read in order to treat close_notify specially.
// The next release of rustls will handle close_notify in a more intuitive way.
impl Read for RustlsStream {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
match self.0.read(buf) {
Ok(size) => Ok(size),
Err(ref e) if is_close_notify(e) => Ok(0),
Err(e) => Err(e),
}
}
}
impl Write for RustlsStream {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.0.write(buf)
}
fn flush(&mut self) -> io::Result<()> {
self.0.flush()
}
}
#[cfg(feature = "native-certs")]
fn root_certs() -> rustls::RootCertStore {
let mut root_store = rustls::RootCertStore::empty();
let certs = rustls_native_certs::load_native_certs().expect("Could not load platform certs");
for cert in certs {
// Repackage the certificate DER bytes.
let rustls_cert = rustls::Certificate(cert.0);
root_store
.add(&rustls_cert)
.expect("Failed to add native certificate too root store");
}
root_store
}
#[cfg(not(feature = "native-certs"))]
fn root_certs() -> rustls::RootCertStore {
let mut root_store = rustls::RootCertStore::empty();
root_store.add_server_trust_anchors(webpki_roots::TLS_SERVER_ROOTS.0.iter().map(|ta| {
rustls::OwnedTrustAnchor::from_subject_spki_name_constraints(
ta.subject,
ta.spki,
ta.name_constraints,
)
}));
root_store
}
impl TlsConnector for Arc<rustls::ClientConfig> {
fn connect(
&self,
dns_name: &str,
mut tcp_stream: TcpStream,
) -> Result<Box<dyn HttpsStream>, Error> {
let sni =
rustls::ServerName::try_from(dns_name).map_err(|e| ErrorKind::Dns.new().src(e))?;
let mut sess = rustls::ClientConnection::new(self.clone(), sni)
.map_err(|e| ErrorKind::Io.new().src(e))?;
sess.complete_io(&mut tcp_stream)
.map_err(|err| ErrorKind::ConnectionFailed.new().src(err))?;
let stream = rustls::StreamOwned::new(sess, tcp_stream);
Ok(Box::new(RustlsStream(stream)))
}
}
pub fn default_tls_config() -> Arc<dyn TlsConnector> {
static TLS_CONF: Lazy<Arc<dyn TlsConnector>> = Lazy::new(|| {
let config = rustls::ClientConfig::builder()
.with_safe_defaults()
.with_root_certificates(root_certs())
.with_no_client_auth();
Arc::new(Arc::new(config))
});
TLS_CONF.clone()
}

242
src/stream.rs
View File

@@ -8,10 +8,6 @@ use std::{fmt, io::Cursor};
use chunked_transfer::Decoder as ChunkDecoder;
#[cfg(feature = "tls")]
use rustls::ClientConnection;
#[cfg(feature = "tls")]
use rustls::StreamOwned;
#[cfg(feature = "socks-proxy")]
use socks::{TargetAddr, ToTargetAddr};
@@ -21,16 +17,83 @@ use crate::{error::Error, proxy::Proto};
use crate::error::ErrorKind;
use crate::unit::Unit;
pub(crate) struct Stream {
inner: BufReader<Inner>,
pub trait HttpsStream: Read + Write + Send + Sync + 'static {
fn socket(&self) -> Option<&TcpStream>;
}
#[allow(clippy::large_enum_variant)]
enum Inner {
Http(TcpStream),
#[cfg(feature = "tls")]
Https(rustls::StreamOwned<rustls::ClientConnection, TcpStream>),
Test(Box<dyn Read + Send + Sync>, Vec<u8>),
pub trait TlsConnector: Send + Sync {
fn connect(
&self,
dns_name: &str,
tcp_stream: TcpStream,
) -> Result<Box<dyn HttpsStream>, crate::error::Error>;
}
pub(crate) struct Stream {
inner: BufReader<Box<dyn Inner + Send + Sync + 'static>>,
}
trait Inner: Read + Write {
fn is_poolable(&self) -> bool;
fn socket(&self) -> Option<&TcpStream>;
fn as_write_vec(&self) -> &[u8] {
panic!("as_write_vec on non Test stream");
}
}
impl<T: HttpsStream + ?Sized> HttpsStream for Box<T> {
fn socket(&self) -> Option<&TcpStream> {
HttpsStream::socket(self.as_ref())
}
}
impl<T: HttpsStream> Inner for T {
fn is_poolable(&self) -> bool {
true
}
fn socket(&self) -> Option<&TcpStream> {
HttpsStream::socket(self)
}
}
impl Inner for TcpStream {
fn is_poolable(&self) -> bool {
true
}
fn socket(&self) -> Option<&TcpStream> {
Some(self)
}
}
struct TestStream(Box<dyn Read + Send + Sync>, Vec<u8>);
impl Inner for TestStream {
fn is_poolable(&self) -> bool {
false
}
fn socket(&self) -> Option<&TcpStream> {
None
}
fn as_write_vec(&self) -> &[u8] {
&self.1
}
}
impl Read for TestStream {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
self.0.read(buf)
}
}
impl Write for TestStream {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.1.write(buf)
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
// DeadlineStream wraps a stream such that read() will return an error
@@ -112,16 +175,20 @@ pub(crate) fn io_err_timeout(error: String) -> io::Error {
impl fmt::Debug for Stream {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self.inner.get_ref() {
Inner::Http(tcpstream) => write!(f, "{:?}", tcpstream),
#[cfg(feature = "tls")]
Inner::Https(tlsstream) => write!(f, "{:?}", tlsstream.get_ref()),
Inner::Test(_, _) => write!(f, "Stream(Test)"),
match self.inner.get_ref().socket() {
Some(s) => write!(f, "{:?}", s),
None => write!(f, "Stream(Test)"),
}
}
}
impl Stream {
fn new(t: impl Inner + Send + Sync + 'static) -> Stream {
Stream::logged_create(Stream {
inner: BufReader::new(Box::new(t)),
})
}
fn logged_create(stream: Stream) -> Stream {
debug!("created stream: {:?}", stream);
stream
@@ -129,20 +196,13 @@ impl Stream {
pub(crate) fn from_vec(v: Vec<u8>) -> Stream {
Stream::logged_create(Stream {
inner: BufReader::new(Inner::Test(Box::new(Cursor::new(v)), vec![])),
inner: BufReader::new(Box::new(TestStream(Box::new(Cursor::new(v)), vec![]))),
})
}
fn from_tcp_stream(t: TcpStream) -> Stream {
Stream::logged_create(Stream {
inner: BufReader::new(Inner::Http(t)),
})
}
#[cfg(feature = "tls")]
fn from_tls_stream(t: StreamOwned<ClientConnection, TcpStream>) -> Stream {
Stream::logged_create(Stream {
inner: BufReader::new(Inner::Https(t)),
inner: BufReader::new(Box::new(t)),
})
}
@@ -186,12 +246,7 @@ impl Stream {
}
}
pub fn is_poolable(&self) -> bool {
match self.inner.get_ref() {
Inner::Http(_) => true,
#[cfg(feature = "tls")]
Inner::Https(_) => true,
_ => false,
}
self.inner.get_ref().is_poolable()
}
pub(crate) fn reset(&mut self) -> io::Result<()> {
@@ -206,12 +261,7 @@ impl Stream {
}
pub(crate) fn socket(&self) -> Option<&TcpStream> {
match self.inner.get_ref() {
Inner::Http(b) => Some(b),
#[cfg(feature = "tls")]
Inner::Https(b) => Some(b.get_ref()),
_ => None,
}
self.inner.get_ref().socket()
}
pub(crate) fn set_read_timeout(&self, timeout: Option<Duration>) -> io::Result<()> {
@@ -223,11 +273,8 @@ impl Stream {
}
#[cfg(test)]
pub fn to_write_vec(&self) -> Vec<u8> {
match self.inner.get_ref() {
Inner::Test(_, writer) => writer.clone(),
_ => panic!("to_write_vec on non Test stream"),
}
pub fn as_write_vec(&self) -> &[u8] {
self.inner.get_ref().as_write_vec()
}
}
@@ -237,17 +284,6 @@ impl Read for Stream {
}
}
impl Read for Inner {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
match self {
Inner::Http(sock) => sock.read(buf),
#[cfg(feature = "tls")]
Inner::Https(stream) => read_https(stream, buf),
Inner::Test(reader, _) => reader.read(buf),
}
}
}
impl BufRead for Stream {
fn fill_buf(&mut self) -> io::Result<&[u8]> {
self.inner.fill_buf()
@@ -268,50 +304,12 @@ where
}
}
#[cfg(feature = "tls")]
fn read_https(
stream: &mut StreamOwned<ClientConnection, TcpStream>,
buf: &mut [u8],
) -> io::Result<usize> {
match stream.read(buf) {
Ok(size) => Ok(size),
Err(ref e) if is_close_notify(e) => Ok(0),
Err(e) => Err(e),
}
}
#[allow(deprecated)]
#[cfg(feature = "tls")]
fn is_close_notify(e: &std::io::Error) -> bool {
if e.kind() != io::ErrorKind::ConnectionAborted {
return false;
}
if let Some(msg) = e.get_ref() {
// :(
return msg.description().contains("CloseNotify");
}
false
}
impl Write for Stream {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
match self.inner.get_mut() {
Inner::Http(sock) => sock.write(buf),
#[cfg(feature = "tls")]
Inner::Https(stream) => stream.write(buf),
Inner::Test(_, writer) => writer.write(buf),
}
self.inner.get_mut().write(buf)
}
fn flush(&mut self) -> io::Result<()> {
match self.inner.get_mut() {
Inner::Http(sock) => sock.flush(),
#[cfg(feature = "tls")]
Inner::Https(stream) => stream.flush(),
Inner::Test(_, writer) => writer.flush(),
}
self.inner.get_mut().flush()
}
}
@@ -328,55 +326,14 @@ pub(crate) fn connect_http(unit: &Unit, hostname: &str) -> Result<Stream, Error>
connect_host(unit, hostname, port).map(Stream::from_tcp_stream)
}
#[cfg(feature = "tls")]
pub(crate) fn connect_https(unit: &Unit, hostname: &str) -> Result<Stream, Error> {
use once_cell::sync::Lazy;
use std::{convert::TryFrom, sync::Arc};
static TLS_CONF: Lazy<Arc<rustls::ClientConfig>> = Lazy::new(|| {
let mut root_store = rustls::RootCertStore::empty();
#[cfg(not(feature = "native-certs"))]
root_store.add_server_trust_anchors(webpki_roots::TLS_SERVER_ROOTS.0.iter().map(|ta| {
rustls::OwnedTrustAnchor::from_subject_spki_name_constraints(
ta.subject,
ta.spki,
ta.name_constraints,
)
}));
#[cfg(feature = "native-certs")]
for cert in rustls_native_certs::load_native_certs().expect("Could not load platform certs")
{
root_store.add(&rustls::Certificate(cert.0)).unwrap();
}
let config = rustls::ClientConfig::builder()
.with_safe_defaults()
.with_root_certificates(root_store)
.with_no_client_auth();
Arc::new(config)
});
let port = unit.url.port().unwrap_or(443);
let tls_conf: Arc<rustls::ClientConfig> = unit
.agent
.config
.tls_config
.as_ref()
.map(|c| c.0.clone())
.unwrap_or_else(|| TLS_CONF.clone());
let mut sock = connect_host(unit, hostname, port)?;
let mut sess = rustls::ClientConnection::new(
tls_conf,
rustls::ServerName::try_from(hostname).map_err(|e| ErrorKind::Dns.new().src(e))?,
)
.map_err(|e| ErrorKind::Io.new().src(e))?;
let sock = connect_host(unit, hostname, port)?;
sess.complete_io(&mut sock)
.map_err(|err| ErrorKind::ConnectionFailed.new().src(err))?;
let stream = rustls::StreamOwned::new(sess, sock);
Ok(Stream::from_tls_stream(stream))
let tls_conf = &unit.agent.config.tls_config;
let https_stream = tls_conf.connect(hostname, sock)?;
Ok(Stream::new(https_stream))
}
pub(crate) fn connect_host(unit: &Unit, hostname: &str, port: u16) -> Result<TcpStream, Error> {
@@ -666,10 +623,3 @@ pub(crate) fn connect_test(unit: &Unit) -> Result<Stream, Error> {
pub(crate) fn connect_test(unit: &Unit) -> Result<Stream, Error> {
Err(ErrorKind::UnknownScheme.msg(&format!("unknown scheme '{}'", unit.url.scheme())))
}
#[cfg(not(feature = "tls"))]
pub(crate) fn connect_https(unit: &Unit, _hostname: &str) -> Result<Stream, Error> {
Err(ErrorKind::UnknownScheme
.msg("URL has 'https:' scheme but ureq was build without HTTP support")
.url(unit.url.clone()))
}

14
src/test/body_send.rs
View File

@@ -10,7 +10,7 @@ fn content_length_on_str() {
let resp = post("test://host/content_length_on_str")
.send_string("Hello World!!!")
.unwrap();
let vec = resp.to_write_vec();
let vec = resp.as_write_vec();
let s = String::from_utf8_lossy(&vec);
assert!(s.contains("\r\nContent-Length: 14\r\n"));
}
@@ -24,7 +24,7 @@ fn user_set_content_length_on_str() {
.set("Content-Length", "12345")
.send_string("Hello World!!!")
.unwrap();
let vec = resp.to_write_vec();
let vec = resp.as_write_vec();
let s = String::from_utf8_lossy(&vec);
assert!(s.contains("\r\nContent-Length: 12345\r\n"));
}
@@ -43,7 +43,7 @@ fn content_length_on_json() {
let resp = post("test://host/content_length_on_json")
.send_json(SerdeValue::Object(json))
.unwrap();
let vec = resp.to_write_vec();
let vec = resp.as_write_vec();
let s = String::from_utf8_lossy(&vec);
assert!(s.contains("\r\nContent-Length: 20\r\n"));
}
@@ -57,7 +57,7 @@ fn content_length_and_chunked() {
.set("Transfer-Encoding", "chunked")
.send_string("Hello World!!!")
.unwrap();
let vec = resp.to_write_vec();
let vec = resp.as_write_vec();
let s = String::from_utf8_lossy(&vec);
assert!(s.contains("Transfer-Encoding: chunked\r\n"));
assert!(!s.contains("\r\nContent-Length:\r\n"));
@@ -73,7 +73,7 @@ fn str_with_encoding() {
.set("Content-Type", "text/plain; charset=iso-8859-1")
.send_string("Hällo Wörld!!!")
.unwrap();
let vec = resp.to_write_vec();
let vec = resp.as_write_vec();
assert_eq!(
&vec[vec.len() - 14..],
//H ä l l o _ W ö r l d ! ! !
@@ -95,7 +95,7 @@ fn content_type_on_json() {
let resp = post("test://host/content_type_on_json")
.send_json(SerdeValue::Object(json))
.unwrap();
let vec = resp.to_write_vec();
let vec = resp.as_write_vec();
let s = String::from_utf8_lossy(&vec);
assert!(s.contains("\r\nContent-Type: application/json\r\n"));
}
@@ -115,7 +115,7 @@ fn content_type_not_overriden_on_json() {
.set("content-type", "text/plain")
.send_json(SerdeValue::Object(json))
.unwrap();
let vec = resp.to_write_vec();
let vec = resp.as_write_vec();
let s = String::from_utf8_lossy(&vec);
assert!(s.contains("\r\ncontent-type: text/plain\r\n"));
}

8
src/test/query_string.rs
View File

@@ -8,7 +8,7 @@ fn no_query_string() {
test::make_response(200, "OK", vec![], vec![])
});
let resp = get("test://host/no_query_string").call().unwrap();
let vec = resp.to_write_vec();
let vec = resp.as_write_vec();
let s = String::from_utf8_lossy(&vec);
assert!(s.contains("GET /no_query_string HTTP/1.1"))
}
@@ -23,7 +23,7 @@ fn escaped_query_string() {
.query("baz", "yo lo")
.call()
.unwrap();
let vec = resp.to_write_vec();
let vec = resp.as_write_vec();
let s = String::from_utf8_lossy(&vec);
assert!(
s.contains("GET /escaped_query_string?foo=bar&baz=yo+lo HTTP/1.1"),
@@ -38,7 +38,7 @@ fn query_in_path() {
test::make_response(200, "OK", vec![], vec![])
});
let resp = get("test://host/query_in_path?foo=bar").call().unwrap();
let vec = resp.to_write_vec();
let vec = resp.as_write_vec();
let s = String::from_utf8_lossy(&vec);
assert!(s.contains("GET /query_in_path?foo=bar HTTP/1.1"))
}
@@ -52,7 +52,7 @@ fn query_in_path_and_req() {
.query("baz", "1 2 3")
.call()
.unwrap();
let vec = resp.to_write_vec();
let vec = resp.as_write_vec();
let s = String::from_utf8_lossy(&vec);
assert!(s.contains("GET /query_in_path_and_req?foo=bar&baz=1+2+3 HTTP/1.1"))
}

40
src/test/range.rs
View File

@@ -1,12 +1,11 @@
#[cfg(feature = "tls")]
use std::io::Read;
#[cfg(feature = "tls")]
use super::super::*;
#[test]
#[cfg(feature = "tls")]
fn read_range() {
fn read_range_rustls() {
use std::io::Read;
use super::super::*;
// rustls is used via crate level convenience calls
let resp = get("https://ureq.s3.eu-central-1.amazonaws.com/sherlock.txt")
.set("Range", "bytes=1000-1999")
.call()
@@ -21,3 +20,30 @@ fn read_range() {
[83, 99, 111, 116, 116, 34, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32]
)
}
#[test]
#[cfg(feature = "native-tls")]
fn read_range_native_tls() {
use std::io::Read;
use std::sync::Arc;
use super::super::*;
let tls_config = native_tls::TlsConnector::new().unwrap();
let agent = builder().tls_connector(Arc::new(tls_config)).build();
let resp = agent
.get("https://ureq.s3.eu-central-1.amazonaws.com/sherlock.txt")
.set("Range", "bytes=1000-1999")
.call()
.unwrap();
assert_eq!(resp.status(), 206);
let mut reader = resp.into_reader();
let mut buf = vec![];
let len = reader.read_to_end(&mut buf).unwrap();
assert_eq!(len, 1000);
assert_eq!(
&buf[0..20],
[83, 99, 111, 116, 116, 34, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32]
)
}

6
src/test/simple.rs
View File

@@ -120,7 +120,7 @@ fn escape_path() {
test::make_response(200, "OK", vec![], vec![])
});
let resp = get("test://host/escape_path here").call().unwrap();
let vec = resp.to_write_vec();
let vec = resp.as_write_vec();
let s = String::from_utf8_lossy(&vec);
assert!(s.contains("GET /escape_path%20here HTTP/1.1"))
}
@@ -198,7 +198,7 @@ pub fn host_no_port() {
test::make_response(200, "OK", vec![], vec![])
});
let resp = get("test://myhost/host_no_port").call().unwrap();
let vec = resp.to_write_vec();
let vec = resp.as_write_vec();
let s = String::from_utf8_lossy(&vec);
assert!(s.contains("\r\nHost: myhost\r\n"));
}
@@ -209,7 +209,7 @@ pub fn host_with_port() {
test::make_response(200, "OK", vec![], vec![])
});
let resp = get("test://myhost:234/host_with_port").call().unwrap();
let vec = resp.to_write_vec();
let vec = resp.as_write_vec();
let s = String::from_utf8_lossy(&vec);
assert!(s.contains("\r\nHost: myhost:234\r\n"));
}

12
test.sh
View File

@@ -4,11 +4,9 @@ set -eu
export RUST_BACKTRACE=1
export RUSTFLAGS="-D dead_code -D unused-variables -D unused"
for tls in "" tls ; do
for feature in "" json charset cookies socks-proxy native-certs ; do
if ! cargo test --no-default-features --features "${tls} ${feature}" ; then
echo Command failed: cargo test \"${what}\" --no-default-features --features \"${tls} ${feature}\"
exit 1
fi
done
for feature in "" tls json charset cookies socks-proxy "tls native-certs" native-tls; do
if ! cargo test --no-default-features --features "${feature}" ; then
echo Command failed: cargo test --no-default-features --features \"${feature}\"
exit 1
fi
done

12
tests/https-agent.rs
View File

@@ -1,5 +1,4 @@
#[cfg(feature = "tls")]
#[cfg(feature = "json")]
#[cfg(all(feature = "json", any(feature = "tls", feature = "tls-native")))]
#[test]
fn agent_set_header() {
use serde::Deserialize;
@@ -23,10 +22,12 @@ fn agent_set_header() {
assert_eq!("value", json.headers.get("Header").unwrap());
}
#[test]
#[cfg(any(feature = "tls", feature = "tls-native"))]
// From here https://badssl.com/download/
// Decrypt key with: openssl rsa -in ./badssl.com-client.pem
#[cfg(feature = "tls")]
const BADSSL_CLIENT_CERT_PEM: &str = r#"Bag Attributes
fn tls_client_certificate() {
const BADSSL_CLIENT_CERT_PEM: &str = r#"Bag Attributes
localKeyID: 41 C3 6C 33 C7 E3 36 DD EA 4A 1F C0 B7 23 B8 E6 9C DC D8 0F
subject=C = US, ST = California, L = San Francisco, O = BadSSL, CN = BadSSL Client Certificate
@@ -91,9 +92,6 @@ m0Wqhhi8/24Sy934t5Txgkfoltg8ahkx934WjP6WWRnSAu+cf+vW
-----END RSA PRIVATE KEY-----
"#;
#[cfg(feature = "tls")]
#[test]
fn tls_client_certificate() {
use ureq::OrAnyStatus;
let certs = rustls_pemfile::certs(&mut BADSSL_CLIENT_CERT_PEM.as_bytes())