feature: use native-tls in ureq requests to trust system TLS certs

pdf2md/cli/Cargo.toml

@@ -7,4 +7,5 @@ edition = "2021"
 base64 = "0.22.1"
 clap = { version = "4.5.20", features = ["derive", "env"] }
 serde_json = "1.0.132"
-ureq = { version = "2.10.1", features = ["json"] }
+ureq = { version = "2.10.1", features = ["json", "native-tls"] }
+native-tls = "0.2.12"

pdf2md/cli/src/operators/create_task.rs

@@ -1,10 +1,15 @@
+use std::sync::Arc;
+
 use base64::Engine;
 
 pub fn create_task(file: &str, base_url: &str, api_key: &str) {
     let file_buf = std::fs::read(file).expect("Failed to read file");
     let file_base64 = base64::prelude::BASE64_STANDARD.encode(file_buf);
 
-    let request = ureq::post(format!("{}/api/task", base_url).as_str())
+    let request = ureq::AgentBuilder::new()
+        .tls_connector(Arc::new(native_tls::TlsConnector::new()?))                                                                    
+        .build()
+        .post(format!("{}/api/task", base_url).as_str())
         .set("Content-Type", "application/json")
         .set("Authorization", api_key)
         .send_json(serde_json::json!({

pdf2md/cli/src/operators/poll_task.rs

@@ -1,6 +1,11 @@
+use std::sync::Arc;
+
 pub fn poll_task(task_id: &str, base_url: &str, api_key: &str) {
     loop {
-        let request = ureq::get(format!("{}/api/task/{}", base_url, task_id).as_str())
+        let request = ureq::AgentBuilder::new()
+            .tls_connector(Arc::new(native_tls::TlsConnector::new()?))                                                                    
+            .build()
+            .get(format!("{}/api/task/{}", base_url, task_id).as_str())
             .set("Content-Type", "application/json")
             .set("Authorization", api_key)
             .call()

server/Cargo.toml

@@ -155,7 +155,8 @@ scraper = "0.19.0"
 regex-split = "0.1.0"
 simple-server-timing-header = "0.1.1"
 ndarray = "0.15.6"
-ureq = { version = "2.9.6", features = ["json"] }
+ureq = { version = "2.9.6", features = ["json", "native-tls"] }
+native-tls = "0.2.12"
 env_logger = "0.11.5"
 tokio-postgres = "0.7.10"
 postgres-openssl = "0.5.0"

server/src/bin/crawl-worker.rs

@@ -608,7 +608,14 @@ async fn crawl(
             let cleaned_url = crawl_request.url.trim_end_matches("/");
             let url = format!("{}/products.json?page={}", cleaned_url, cur_page);
 
-            let response: ShopifyResponse = ureq::get(&url)
+            let response: ShopifyResponse = ureq::AgentBuilder::new()
+                .tls_connector(Arc::new(native_tls::TlsConnector::new().map_err(|_| {
+                    ServiceError::InternalServerError(
+                        "Failed to acquire tls connection".to_string()
+                    )
+                })?))
+                .build()
+                .get(&url)
                 .call()
                 .map_err(|e| ServiceError::InternalServerError(format!("Failed to fetch: {}", e)))?
                 .into_json()