Revert diff
This commit is contained in:
committed by
Martin Algesten
parent
3cd160279f
commit
b20c4fc3be
@@ -1,11 +1,11 @@
|
||||
use std::io::{self, BufRead, BufReader, Read, Write};
|
||||
use std::io::{self, BufRead, BufReader, Read};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::time::Duration;
|
||||
use std::{env, error, fmt, result, result::Result};
|
||||
use std::{env, error, fmt, result};
|
||||
|
||||
use log::{error, info};
|
||||
use ureq::{self, Response};
|
||||
use ureq;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Oops(String);
|
||||
@@ -40,36 +40,12 @@ fn get(agent: &ureq::Agent, url: &str) -> Result<Vec<u8>> {
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn get_and_write(agent: &ureq::Agent, url: &str) -> std::result::Result<(), Oops> {
|
||||
let r = get_response(agent, url)?;
|
||||
let mut reader = r.into_reader();
|
||||
let mut bytes = vec![];
|
||||
reader.read_to_end(&mut bytes)?;
|
||||
std::io::stdout().write_all(&bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_response(agent: &ureq::Agent, url: &str) -> result::Result<Response, ureq::Error> {
|
||||
let fetch = || agent.get(url).call();
|
||||
let mut result = fetch();
|
||||
for _ in 1..4 {
|
||||
match result {
|
||||
Err(ureq::Error {
|
||||
response: Some(r), ..
|
||||
}) if r.status() == 429 => {
|
||||
let retry: u64 = r
|
||||
.header("retry-after")
|
||||
.and_then(|h| h.parse().ok())
|
||||
.unwrap_or(5);
|
||||
eprintln!("403: {}", r.into_string()?);
|
||||
thread::sleep(Duration::from_secs(retry));
|
||||
fn get_and_write(agent: &ureq::Agent, url: &str) {
|
||||
info!("🕷️ {}", url);
|
||||
match get(agent, url) {
|
||||
Ok(_) => info!("✔️ {}", url),
|
||||
Err(e) => error!("⚠️ {} {}", url, e),
|
||||
}
|
||||
r => return r,
|
||||
}
|
||||
result = fetch();
|
||||
}
|
||||
println!("Failed after 5 tries: {:?}", &result);
|
||||
result
|
||||
}
|
||||
|
||||
fn get_many(urls: Vec<String>, simultaneous_fetches: usize) -> Result<()> {
|
||||
|
||||
Reference in New Issue
Block a user