diff --git a/examples/smoke-test/main.rs b/examples/smoke-test/main.rs index ad729f4..6f0e368 100644 --- a/examples/smoke-test/main.rs +++ b/examples/smoke-test/main.rs @@ -61,7 +61,7 @@ fn get_and_write(agent: &ureq::Agent, url: &String) -> Result<()> { } fn get_many(urls: Vec, simultaneous_fetches: usize) -> Result<()> { - let agent = ureq::Agent::default().build(); + let agent = ureq::Agent::default(); let pool = rayon::ThreadPoolBuilder::new() .num_threads(simultaneous_fetches) .build()?; diff --git a/src/agent.rs b/src/agent.rs index c6b7a10..154b8a1 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -13,6 +13,25 @@ use crate::proxy::Proxy; use crate::request::Request; use crate::resolve::ArcResolver; +#[derive(Debug, Default)] +pub struct AgentBuilder { + headers: Vec
, + proxy: Option, + max_idle_connections: usize, + max_idle_connections_per_host: usize, + /// Cookies saved between requests. + /// Invariant: All cookies must have a nonempty domain and path. + #[cfg(feature = "cookie")] + jar: CookieStore, + resolver: ArcResolver, +} + +impl Default for Agent { + fn default() -> Self { + AgentBuilder::new().build() + } +} + /// Agents keep state between requests. /// /// By default, no state, such as cookies, is kept between requests. @@ -41,7 +60,7 @@ use crate::resolve::ArcResolver; /// println!("Secret is: {}", secret.unwrap().into_string().unwrap()); /// } /// ``` -#[derive(Debug, Default, Clone)] +#[derive(Debug, Clone)] pub struct Agent { /// Copied into each request of this agent. pub(crate) headers: Vec
, @@ -65,98 +84,12 @@ pub(crate) struct AgentState { } impl AgentState { - fn new() -> Self { - Self::default() - } - pub fn pool(&mut self) -> &mut ConnectionPool { + pub(crate) fn pool(&mut self) -> &mut ConnectionPool { &mut self.pool } } impl Agent { - /// Creates a new agent. Typically you'd use [`ureq::agent()`](fn.agent.html) to - /// do this. - /// - /// ``` - /// let agent = ureq::Agent::new() - /// .set("X-My-Header", "Foo") // present on all requests from this agent - /// .build(); - /// - /// agent.get("/foo"); - /// ``` - pub fn new() -> Agent { - Default::default() - } - - /// Create a new agent after treating it as a builder. - /// This actually clones the internal state to a new one and instantiates - /// a new connection pool that is reused between connects. - pub fn build(&self) -> Self { - Agent { - headers: self.headers.clone(), - state: Arc::new(Mutex::new(AgentState::new())), - } - } - - /// Set a header field that will be present in all requests using the agent. - /// - /// ``` - /// let agent = ureq::agent() - /// .set("X-API-Key", "foobar") - /// .set("Accept", "text/plain") - /// .build(); - /// - /// let r = agent - /// .get("/my-page") - /// .call(); - /// - /// if let Ok(resp) = r { - /// println!("yay got {}", resp.into_string().unwrap()); - /// } else { - /// println!("Oh no error!"); - /// } - /// ``` - pub fn set(&mut self, header: &str, value: &str) -> &mut Agent { - header::add_header(&mut self.headers, Header::new(header, value)); - self - } - - /// Basic auth that will be present in all requests using the agent. - /// - /// ``` - /// let agent = ureq::agent() - /// .auth("martin", "rubbermashgum") - /// .build(); - /// - /// let r = agent - /// .get("/my_page") - /// .call(); - /// println!("{:?}", r); - /// ``` - pub fn auth(&mut self, user: &str, pass: &str) -> &mut Agent { - let pass = basic_auth(user, pass); - self.auth_kind("Basic", &pass) - } - - /// Auth of other kinds such as `Digest`, `Token` etc, that will be present - /// in all requests using the agent. - /// - /// ``` - /// // sets a header "Authorization: token secret" - /// let agent = ureq::agent() - /// .auth_kind("token", "secret") - /// .build(); - /// - /// let r = agent - /// .get("/my_page") - /// .call(); - /// ``` - pub fn auth_kind(&mut self, kind: &str, pass: &str) -> &mut Agent { - let value = format!("{} {}", kind, pass); - self.set("Authorization", &value); - self - } - /// Request by providing the HTTP verb such as `GET`, `POST`... /// /// ``` @@ -171,73 +104,6 @@ impl Agent { Request::new(&self, method.into(), path.into()) } - /// Sets the maximum number of connections allowed in the connection pool. - /// By default, this is set to 100. Setting this to zero would disable - /// connection pooling. - /// - /// ``` - /// let agent = ureq::agent(); - /// agent.set_max_pool_connections(200); - /// ``` - pub fn set_max_pool_connections(&self, max_connections: usize) { - let mut state = self.state.lock().unwrap(); - state.pool.set_max_idle_connections(max_connections); - } - - /// Sets the maximum number of connections per host to keep in the - /// connection pool. By default, this is set to 1. Setting this to zero - /// would disable connection pooling. - /// - /// ``` - /// let agent = ureq::agent(); - /// agent.set_max_pool_connections_per_host(10); - /// ``` - pub fn set_max_pool_connections_per_host(&self, max_connections: usize) { - let mut state = self.state.lock().unwrap(); - state - .pool - .set_max_idle_connections_per_host(max_connections); - } - - /// Configures a custom resolver to be used by this agent. By default, - /// address-resolution is done by std::net::ToSocketAddrs. This allows you - /// to override that resolution with your own alternative. Useful for - /// testing and special-cases like DNS-based load balancing. - /// - /// A `Fn(&str) -> io::Result>` is a valid resolver, - /// passing a closure is a simple way to override. Note that you might need - /// explicit type `&str` on the closure argument for type inference to - /// succeed. - /// ``` - /// use std::net::ToSocketAddrs; - /// - /// let mut agent = ureq::agent(); - /// agent.set_resolver(|addr: &str| match addr { - /// "example.com" => Ok(vec![([127,0,0,1], 8096).into()]), - /// addr => addr.to_socket_addrs().map(Iterator::collect), - /// }); - /// ``` - pub fn set_resolver(&mut self, resolver: impl crate::Resolver + 'static) -> &mut Self { - self.state.lock().unwrap().resolver = resolver.into(); - self - } - - /// Set the proxy server to use for all connections from this Agent. - /// - /// Example: - /// ``` - /// let proxy = ureq::Proxy::new("user:password@cool.proxy:9090").unwrap(); - /// let agent = ureq::agent() - /// .set_proxy(proxy) - /// .build(); - /// ``` - pub fn set_proxy(&mut self, proxy: Proxy) -> &mut Agent { - let mut state = self.state.lock().unwrap(); - state.proxy = Some(proxy); - drop(state); - self - } - /// Gets a cookie in this agent by name. Cookies are available /// either by setting it in the agent, or by making requests /// that `Set-Cookie` in the agent. @@ -345,6 +211,156 @@ impl Agent { } } +impl AgentBuilder { + pub fn new() -> AgentBuilder { + AgentBuilder { + max_idle_connections: 100, + max_idle_connections_per_host: 1, + ..Default::default() + } + } + + /// Create a new agent. + // Note: This could take &self as the first argument, allowing one + // AgentBuilder to be used multiple times, except CookieStore does + // not implement clone, so we have to give ownership to the newly + // built Agent. + pub fn build(self) -> Agent { + Agent { + headers: self.headers.clone(), + state: Arc::new(Mutex::new(AgentState { + pool: ConnectionPool::new( + self.max_idle_connections, + self.max_idle_connections_per_host, + ), + proxy: self.proxy.clone(), + #[cfg(feature = "cookie")] + jar: self.jar, + resolver: self.resolver, + })), + } + } + + /// Set a header field that will be present in all requests using the agent. + /// + /// ``` + /// let agent = ureq::AgentBuilder::new() + /// .set("X-API-Key", "foobar") + /// .set("Accept", "text/plain") + /// .build(); + /// + /// let r = agent + /// .get("/my-page") + /// .call(); + /// + /// if let Ok(resp) = r { + /// println!("yay got {}", resp.into_string().unwrap()); + /// } else { + /// println!("Oh no error!"); + /// } + /// ``` + pub fn set(mut self, header: &str, value: &str) -> Self { + header::add_header(&mut self.headers, Header::new(header, value)); + self + } + + /// Basic auth that will be present in all requests using the agent. + /// + /// ``` + /// let agent = ureq::AgentBuilder::new() + /// .auth("martin", "rubbermashgum") + /// .build(); + /// + /// let r = agent + /// .get("/my_page") + /// .call(); + /// println!("{:?}", r); + /// ``` + pub fn auth(self, user: &str, pass: &str) -> Self { + let pass = basic_auth(user, pass); + self.auth_kind("Basic", &pass) + } + + /// Auth of other kinds such as `Digest`, `Token` etc, that will be present + /// in all requests using the agent. + /// + /// ``` + /// // sets a header "Authorization: token secret" + /// let agent = ureq::AgentBuilder::new() + /// .auth_kind("token", "secret") + /// .build(); + /// + /// let r = agent + /// .get("/my_page") + /// .call(); + /// ``` + pub fn auth_kind(self, kind: &str, pass: &str) -> Self { + let value = format!("{} {}", kind, pass); + self.set("Authorization", &value) + } + /// Sets the maximum number of connections allowed in the connection pool. + /// By default, this is set to 100. Setting this to zero would disable + /// connection pooling. + /// + /// ``` + /// let agent = ureq::AgentBuilder::new().max_pool_connections(200).build(); + /// ``` + pub fn max_pool_connections(mut self, max: usize) -> Self { + self.max_idle_connections = max; + self + } + + /// Sets the maximum number of connections per host to keep in the + /// connection pool. By default, this is set to 1. Setting this to zero + /// would disable connection pooling. + /// + /// ``` + /// let agent = ureq::AgentBuilder::new().max_pool_connections_per_host(200).build(); + /// ``` + pub fn max_pool_connections_per_host(mut self, max: usize) -> Self { + self.max_idle_connections_per_host = max; + self + } + + /// Configures a custom resolver to be used by this agent. By default, + /// address-resolution is done by std::net::ToSocketAddrs. This allows you + /// to override that resolution with your own alternative. Useful for + /// testing and special-cases like DNS-based load balancing. + /// + /// A `Fn(&str) -> io::Result>` is a valid resolver, + /// passing a closure is a simple way to override. Note that you might need + /// explicit type `&str` on the closure argument for type inference to + /// succeed. + /// ``` + /// use std::net::ToSocketAddrs; + /// + /// let mut agent = ureq::AgentBuilder::new() + /// .resolver(|addr: &str| match addr { + /// "example.com" => Ok(vec![([127,0,0,1], 8096).into()]), + /// addr => addr.to_socket_addrs().map(Iterator::collect), + /// }) + /// .build(); + /// ``` + pub fn resolver(mut self, resolver: impl crate::Resolver + 'static) -> Self { + self.resolver = resolver.into(); + self + } + + /// Set the proxy server to use for all connections from this Agent. + /// + /// Example: + /// ``` + /// let proxy = ureq::Proxy::new("user:password@cool.proxy:9090").unwrap(); + /// let agent = ureq::AgentBuilder::new() + /// .proxy(proxy) + /// .build(); + /// ``` + pub fn proxy(mut self, proxy: Proxy) -> Self { + self.proxy = Some(proxy); + self + } +} + pub(crate) fn basic_auth(user: &str, pass: &str) -> String { let safe = match user.find(':') { Some(idx) => &user[..idx], @@ -356,16 +372,13 @@ pub(crate) fn basic_auth(user: &str, pass: &str) -> String { #[cfg(test)] mod tests { use super::*; - use std::thread; ///////////////////// AGENT TESTS ////////////////////////////// #[test] - fn agent_implements_send() { - let mut agent = Agent::new(); - thread::spawn(move || { - agent.set("Foo", "Bar"); - }); + fn agent_implements_send_and_sync() { + let _agent: Box = Box::new(AgentBuilder::new().build()); + let _agent: Box = Box::new(AgentBuilder::new().build()); } #[test] @@ -395,15 +408,4 @@ mod tests { let mut buf = vec![]; reader.read_to_end(&mut buf).unwrap(); } - - //////////////////// REQUEST TESTS ///////////////////////////// - - #[test] - fn request_implements_send() { - let agent = Agent::new(); - let mut request = Request::new(&agent, "GET".to_string(), "/foo".to_string()); - thread::spawn(move || { - request.set("Foo", "Bar"); - }); - } } diff --git a/src/lib.rs b/src/lib.rs index 07d2e8b..80a854d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -120,6 +120,7 @@ mod serde_macros; mod test; pub use crate::agent::Agent; +pub use crate::agent::AgentBuilder; pub use crate::error::Error; pub use crate::header::Header; pub use crate::proxy::Proxy; @@ -135,7 +136,7 @@ pub use serde_json::{to_value as serde_to_value, Map as SerdeMap, Value as Serde /// Agents are used to keep state between requests. pub fn agent() -> Agent { - Agent::new().build() + Agent::default() } /// Make a request setting the HTTP method via a string. @@ -144,7 +145,7 @@ pub fn agent() -> Agent { /// ureq::request("GET", "http://example.com").call().unwrap(); /// ``` pub fn request(method: &str, path: &str) -> Request { - Agent::new().request(method, path) + agent().request(method, path) } /// Make a GET request. diff --git a/src/pool.rs b/src/pool.rs index 711edab..30ff980 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -73,16 +73,13 @@ impl Default for ConnectionPool { } impl ConnectionPool { - pub fn set_max_idle_connections(&mut self, max_connections: usize) { - if self.max_idle_connections == max_connections { - return; + pub(crate) fn new(max_idle_connections: usize, max_idle_connections_per_host: usize) -> Self { + ConnectionPool { + recycle: Default::default(), + lru: Default::default(), + max_idle_connections, + max_idle_connections_per_host, } - - // Remove any extra connections if the number was decreased. - while self.lru.len() > max_connections { - self.remove_oldest(); - } - self.max_idle_connections = max_connections; } /// Return true if either of the max_* settings is 0, meaning we should do no work. @@ -90,30 +87,6 @@ impl ConnectionPool { self.max_idle_connections == 0 || self.max_idle_connections_per_host == 0 } - pub fn set_max_idle_connections_per_host(&mut self, max_connections: usize) { - if self.max_idle_connections_per_host == max_connections { - return; - } - - if max_connections == 0 { - // Clear the connection pool, caching is disabled. - self.lru.clear(); - self.recycle.clear(); - return; - } - - // Remove any extra streams if the number was decreased. - for (key, val) in self.recycle.iter_mut() { - while val.len() > max_connections { - // Remove the oldest entry - val.pop_front(); - remove_first_match(&mut self.lru, key) - .expect("invariant failed: key in recycle but not in lru"); - } - } - self.max_idle_connections_per_host = max_connections; - } - /// How the unit::connect tries to get a pooled connection. pub fn try_get_connection(&mut self, url: &Url, proxy: &Option) -> Option { let key = PoolKey::new(url, proxy); @@ -287,50 +260,6 @@ fn pool_per_host_connections_limit() { assert_eq!(pool.len(), 0); } -#[test] -fn pool_update_connection_limit() { - let mut pool = ConnectionPool::default(); - pool.set_max_idle_connections(50); - - let hostnames = (0..pool.max_idle_connections).map(|i| format!("{}.example", i)); - let poolkeys = hostnames.map(|hostname| PoolKey { - scheme: "https".to_string(), - hostname, - port: Some(999), - proxy: None, - }); - for key in poolkeys.clone() { - pool.add(key, Stream::Cursor(std::io::Cursor::new(vec![]))); - } - assert_eq!(pool.len(), 50); - pool.set_max_idle_connections(25); - assert_eq!(pool.len(), 25); -} - -#[test] -fn pool_update_per_host_connection_limit() { - let mut pool = ConnectionPool::default(); - pool.set_max_idle_connections(50); - pool.set_max_idle_connections_per_host(50); - - let poolkey = PoolKey { - scheme: "https".to_string(), - hostname: "example.com".to_string(), - port: Some(999), - proxy: None, - }; - - for _ in 0..50 { - pool.add( - poolkey.clone(), - Stream::Cursor(std::io::Cursor::new(vec![])), - ); - } - assert_eq!(pool.len(), 50); - pool.set_max_idle_connections_per_host(25); - assert_eq!(pool.len(), 25); -} - #[test] fn pool_checks_proxy() { // Test inserting different poolkeys with same address but different proxies. diff --git a/src/request.rs b/src/request.rs index 1423eef..c431144 100644 --- a/src/request.rs +++ b/src/request.rs @@ -690,3 +690,17 @@ fn no_hostname() { ); assert!(req.get_host().is_err()); } + +#[test] +fn request_implements_send_and_sync() { + let _request: Box = Box::new(Request::new( + &Agent::default(), + "GET".to_string(), + "https://example.com/".to_string(), + )); + let _request: Box = Box::new(Request::new( + &Agent::default(), + "GET".to_string(), + "https://example.com/".to_string(), + )); +} diff --git a/src/test/agent_test.rs b/src/test/agent_test.rs index 1c0eee5..08700cb 100644 --- a/src/test/agent_test.rs +++ b/src/test/agent_test.rs @@ -10,7 +10,9 @@ use super::super::*; #[test] fn agent_reuse_headers() { - let agent = agent().set("Authorization", "Foo 12345").build(); + let agent = AgentBuilder::new() + .set("Authorization", "Foo 12345") + .build(); test::set_handler("/agent_reuse_headers", |unit| { assert!(unit.has("Authorization")); @@ -44,7 +46,7 @@ fn idle_timeout_handler(mut stream: TcpStream) -> io::Result<()> { fn connection_reuse() { let testserver = TestServer::new(idle_timeout_handler); let url = format!("http://localhost:{}", testserver.port); - let agent = Agent::default().build(); + let agent = Agent::default(); let resp = agent.get(&url).call().unwrap(); // use up the connection so it gets returned to the pool @@ -87,8 +89,9 @@ fn custom_resolver() { buf }); - crate::agent() - .set_resolver(move |_: &str| Ok(vec![local_addr])) + AgentBuilder::new() + .resolver(move |_: &str| Ok(vec![local_addr])) + .build() .get("http://cool.server/") .call() .ok(); @@ -144,7 +147,7 @@ fn cookie_and_redirect(mut stream: TcpStream) -> io::Result<()> { fn test_cookies_on_redirect() -> Result<(), Error> { let testserver = TestServer::new(cookie_and_redirect); let url = format!("http://localhost:{}/first", testserver.port); - let agent = Agent::default().build(); + let agent = Agent::default(); agent.post(&url).call()?; assert!(agent.cookie("first").is_some()); assert!(agent.cookie("second").is_some()); @@ -168,7 +171,7 @@ fn dirty_streams_not_returned() -> Result<(), Error> { Ok(()) }); let url = format!("http://localhost:{}/", testserver.port); - let agent = Agent::default().build(); + let agent = Agent::default(); let resp = agent.get(&url).call()?; let resp_str = resp.into_string()?; assert_eq!(resp_str, "corgidachsund"); diff --git a/src/test/auth.rs b/src/test/auth.rs index e7fedba..677e165 100644 --- a/src/test/auth.rs +++ b/src/test/auth.rs @@ -55,7 +55,7 @@ fn url_auth_overridden() { ); test::make_response(200, "OK", vec![], vec![]) }); - let agent = agent().auth("martin", "rubbermashgum").build(); + let agent = AgentBuilder::new().auth("martin", "rubbermashgum").build(); let resp = agent .get("test://Aladdin:OpenSesame@host/url_auth_overridden") .call() diff --git a/src/test/timeout.rs b/src/test/timeout.rs index 645d039..edd129f 100644 --- a/src/test/timeout.rs +++ b/src/test/timeout.rs @@ -25,7 +25,7 @@ fn dribble_body_respond(mut stream: TcpStream, contents: &[u8]) -> io::Result<() } fn get_and_expect_timeout(url: String) { - let agent = Agent::default().build(); + let agent = Agent::default(); let timeout = Duration::from_millis(500); let resp = agent.get(&url).timeout(timeout).call().unwrap(); @@ -86,7 +86,7 @@ fn overall_timeout_reading_json() { }); let url = format!("http://localhost:{}/", server.port); - let agent = Agent::default().build(); + let agent = Agent::default(); let timeout = Duration::from_millis(500); let resp = agent.get(&url).timeout(timeout).call().unwrap(); diff --git a/tests/https-agent.rs b/tests/https-agent.rs index faa4589..e23f3c9 100644 --- a/tests/https-agent.rs +++ b/tests/https-agent.rs @@ -102,7 +102,7 @@ m0Wqhhi8/24Sy934t5Txgkfoltg8ahkx934WjP6WWRnSAu+cf+vW #[cfg(feature = "tls")] #[test] fn tls_client_certificate() { - let agent = ureq::Agent::default().build(); + let agent = ureq::Agent::default(); let mut tls_config = rustls::ClientConfig::new();