Revert diff

This commit is contained in:
Jacob Hoffman-Andrews
2020-11-26 13:05:52 -08:00
committed by Martin Algesten
parent 3cd160279f
commit b20c4fc3be

View File

@@ -1,11 +1,11 @@
use std::io::{self, BufRead, BufReader, Read, Write}; use std::io::{self, BufRead, BufReader, Read};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use std::thread::{self, JoinHandle}; use std::thread::{self, JoinHandle};
use std::time::Duration; use std::time::Duration;
use std::{env, error, fmt, result, result::Result}; use std::{env, error, fmt, result};
use log::{error, info}; use log::{error, info};
use ureq::{self, Response}; use ureq;
#[derive(Debug)] #[derive(Debug)]
struct Oops(String); struct Oops(String);
@@ -40,36 +40,12 @@ fn get(agent: &ureq::Agent, url: &str) -> Result<Vec<u8>> {
Ok(bytes) Ok(bytes)
} }
fn get_and_write(agent: &ureq::Agent, url: &str) -> std::result::Result<(), Oops> { fn get_and_write(agent: &ureq::Agent, url: &str) {
let r = get_response(agent, url)?; info!("🕷️ {}", url);
let mut reader = r.into_reader(); match get(agent, url) {
let mut bytes = vec![]; Ok(_) => info!("✔️ {}", url),
reader.read_to_end(&mut bytes)?; Err(e) => error!("⚠️ {} {}", url, e),
std::io::stdout().write_all(&bytes)?;
Ok(())
} }
fn get_response(agent: &ureq::Agent, url: &str) -> result::Result<Response, ureq::Error> {
let fetch = || agent.get(url).call();
let mut result = fetch();
for _ in 1..4 {
match result {
Err(ureq::Error {
response: Some(r), ..
}) if r.status() == 429 => {
let retry: u64 = r
.header("retry-after")
.and_then(|h| h.parse().ok())
.unwrap_or(5);
eprintln!("403: {}", r.into_string()?);
thread::sleep(Duration::from_secs(retry));
}
r => return r,
}
result = fetch();
}
println!("Failed after 5 tries: {:?}", &result);
result
} }
fn get_many(urls: Vec<String>, simultaneous_fetches: usize) -> Result<()> { fn get_many(urls: Vec<String>, simultaneous_fetches: usize) -> Result<()> {