From 43836c57229f96f8e6de99effcd1236d92e489e9 Mon Sep 17 00:00:00 2001 From: MTRNord Date: Thu, 6 Apr 2023 21:46:09 +0200 Subject: [PATCH 1/2] Reuse Client when checking urls and add timeout for requests --- components/link_checker/src/lib.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/components/link_checker/src/lib.rs b/components/link_checker/src/lib.rs index 3b3551afd6..aedada0735 100644 --- a/components/link_checker/src/lib.rs +++ b/components/link_checker/src/lib.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::result; use std::sync::{Arc, RwLock}; +use std::time::Duration; use libs::once_cell::sync::Lazy; use libs::reqwest::header::{HeaderMap, ACCEPT}; @@ -30,6 +31,13 @@ pub fn message(res: &Result) -> String { // Keep history of link checks so a rebuild doesn't have to check again static LINKS: Lazy>>> = Lazy::new(|| Arc::new(RwLock::new(HashMap::new()))); +// Make sure to create only a single Client so that we can reuse the connections +static CLIENT: Lazy = Lazy::new(|| { + Client::builder() + .user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"))) + .build() + .expect("reqwest client build") +}); pub fn check_url(url: &str, config: &LinkChecker) -> Result { { @@ -44,15 +52,11 @@ pub fn check_url(url: &str, config: &LinkChecker) -> Result { headers.append(ACCEPT, "*/*".parse().unwrap()); // TODO: pass the client to the check_url, do not pass the config - let client = Client::builder() - .user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"))) - .build() - .expect("reqwest client build"); let check_anchor = !config.skip_anchor_prefixes.iter().any(|prefix| url.starts_with(prefix)); // Need to actually do the link checking - let res = match client.get(url).headers(headers).send() { + let res = match CLIENT.get(url).headers(headers).send() { Ok(ref mut response) if check_anchor && has_anchor(url) => { let body = { let mut buf: Vec = vec![]; From 605d872bd03475abb4d6e9c634b06bb4e00b4f0f Mon Sep 17 00:00:00 2001 From: MTRNord Date: Thu, 6 Apr 2023 21:52:56 +0200 Subject: [PATCH 2/2] Remove stray use --- components/link_checker/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/components/link_checker/src/lib.rs b/components/link_checker/src/lib.rs index aedada0735..ab44259217 100644 --- a/components/link_checker/src/lib.rs +++ b/components/link_checker/src/lib.rs @@ -1,7 +1,6 @@ use std::collections::HashMap; use std::result; use std::sync::{Arc, RwLock}; -use std::time::Duration; use libs::once_cell::sync::Lazy; use libs::reqwest::header::{HeaderMap, ACCEPT};