diff --git a/examples/smoke-test/main.rs b/examples/smoke-test/main.rs index 4c7ea28..8f1693b 100644 --- a/examples/smoke-test/main.rs +++ b/examples/smoke-test/main.rs @@ -1,11 +1,11 @@ -use std::io::{self, BufRead, BufReader, Read}; +use std::io::{self, BufRead, BufReader, Read, Write}; use std::sync::{Arc, Mutex}; use std::thread::{self, JoinHandle}; use std::time::Duration; -use std::{env, error, fmt, result}; +use std::{env, error, fmt, result, result::Result}; use log::{error, info}; -use ureq; +use ureq::{self, Response}; #[derive(Debug)] struct Oops(String); @@ -40,12 +40,36 @@ fn get(agent: &ureq::Agent, url: &str) -> Result> { Ok(bytes) } -fn get_and_write(agent: &ureq::Agent, url: &str) { - info!("🕷️ {}", url); - match get(agent, url) { - Ok(_) => info!("✔️ {}", url), - Err(e) => error!("⚠️ {} {}", url, e), +fn get_and_write(agent: &ureq::Agent, url: &str) -> std::result::Result<(), Oops> { + let r = get_response(agent, url)?; + let mut reader = r.into_reader(); + let mut bytes = vec![]; + reader.read_to_end(&mut bytes)?; + std::io::stdout().write_all(&bytes)?; + Ok(()) +} + +fn get_response(agent: &ureq::Agent, url: &str) -> result::Result { + let fetch = || agent.get(url).call(); + let mut result = fetch(); + for _ in 1..4 { + match result { + Err(ureq::Error { + response: Some(r), .. + }) if r.status() == 429 => { + let retry: u64 = r + .header("retry-after") + .and_then(|h| h.parse().ok()) + .unwrap_or(5); + eprintln!("403: {}", r.into_string()?); + thread::sleep(Duration::from_secs(retry)); + } + r => return r, + } + result = fetch(); } + println!("Failed after 5 tries: {:?}", &result); + result } fn get_many(urls: Vec, simultaneous_fetches: usize) -> Result<()> { diff --git a/src/error.rs b/src/error.rs index 5e6838b..eebaf04 100644 --- a/src/error.rs +++ b/src/error.rs @@ -44,7 +44,7 @@ pub struct Error { message: Option, url: Option, source: Option>, - response: Option, + pub response: Option, } impl Display for Error {