From a6e99c8b3608ec7d3ef6625149b45a47215c4cd9 Mon Sep 17 00:00:00 2001
From: Jacob Hoffman-Andrews <github@hoffman-andrews.com>
Date: Tue, 23 Jun 2020 21:09:09 -0700
Subject: [PATCH] Add smoke test in examples. (#85)

This loads a list of top domain names (e.g. from
https://tranco-list.eu/) and tries to fetch them all, in parallel. This
can be used to exercise ureq and find panics.
---
 Cargo.toml                  |   6 ++
 examples/smoke-test/main.rs | 107 ++++++++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+)
 create mode 100644 examples/smoke-test/main.rs

diff --git a/Cargo.toml b/Cargo.toml
index ed99436..4071b67 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -41,3 +41,9 @@ native-tls = { version = "0.2", optional = true }
 
 [dev-dependencies]
 serde = { version = "1", features = ["derive"] }
+rayon = "1.3.0"
+rayon-core = "1.7.0"
+chrono = "0.4.11"
+
+[[example]]
+name = "smoke-test"
diff --git a/examples/smoke-test/main.rs b/examples/smoke-test/main.rs
new file mode 100644
index 0000000..39770a7
--- /dev/null
+++ b/examples/smoke-test/main.rs
@@ -0,0 +1,107 @@
+use chrono::Local;
+use rayon::prelude::*;
+use rayon_core;
+
+use std::io::{self, BufRead, BufReader, Read};
+use std::iter::Iterator;
+use std::time::Duration;
+use std::{env, error, fmt, result};
+
+use ureq;
+
+#[derive(Debug)]
+struct Oops(String);
+
+impl From<io::Error> for Oops {
+    fn from(e: io::Error) -> Oops {
+        Oops(e.to_string())
+    }
+}
+
+impl From<&ureq::Error> for Oops {
+    fn from(e: &ureq::Error) -> Oops {
+        Oops(e.to_string())
+    }
+}
+
+impl From<rayon_core::ThreadPoolBuildError> for Oops {
+    fn from(e: rayon_core::ThreadPoolBuildError) -> Oops {
+        Oops(e.to_string())
+    }
+}
+
+impl error::Error for Oops {}
+
+impl fmt::Display for Oops {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+type Result<T> = result::Result<T, Oops>;
+
+fn get(agent: &ureq::Agent, url: &String) -> Result<Vec<u8>> {
+    let response = agent
+        .get(url)
+        .timeout_connect(5_000)
+        .timeout(Duration::from_secs(20))
+        .call();
+    if let Some(err) = response.synthetic_error() {
+        return Err(err.into());
+    }
+    let mut reader = response.into_reader();
+    let mut bytes = vec![];
+    reader.read_to_end(&mut bytes)?;
+    Ok(bytes)
+}
+
+fn get_and_write(agent: &ureq::Agent, url: &String) -> Result<()> {
+    println!("🕷️ {} {}", Local::now(), url);
+    match get(agent, url) {
+        Ok(_) => println!("✔️ {} {}", Local::now(), url),
+        Err(e) => println!("⚠️ {} {} {}", Local::now(), url, e),
+    }
+    Ok(())
+}
+
+fn get_many(urls: Vec<String>, simultaneous_fetches: usize) -> Result<()> {
+    let agent = ureq::Agent::default().build();
+    let pool = rayon::ThreadPoolBuilder::new()
+        .num_threads(simultaneous_fetches)
+        .build()?;
+    pool.scope(|_| {
+        urls.par_iter().map(|u| get_and_write(&agent, u)).count();
+    });
+    Ok(())
+}
+
+fn main() -> Result<()> {
+    let args = env::args();
+    if args.len() == 1 {
+        println!(
+            r##"Usage: {:#?} top-1m.csv
+        
+Where top-1m.csv is a simple, unquoted CSV containing two fields, a rank and
+a domain name. For instance you can get such a list from https://tranco-list.eu/.
+
+For each domain, this program will attempt to GET four URLs: The domain name
+name with HTTP and HTTPS, and with and without a www prefix. It will fetch
+using 50 threads concurrently.
+"##,
+            env::current_exe()?
+        );
+        return Ok(());
+    }
+    let file = std::fs::File::open(args.skip(1).next().unwrap())?;
+    let bufreader = BufReader::new(file);
+    let mut urls = vec![];
+    for line in bufreader.lines() {
+        let domain = line?.rsplit(",").next().unwrap().to_string();
+        urls.push(format!("http://{}/", domain));
+        urls.push(format!("https://{}/", domain));
+        urls.push(format!("http://www.{}/", domain));
+        urls.push(format!("https://www.{}/", domain));
+    }
+    get_many(urls, 50)?;
+    Ok(())
+}