Skip to content

Commit

Permalink
feat(web-scraping): add support for custom HTTP request headers in we…
Browse files Browse the repository at this point in the history 
…b page trackers
  • Loading branch information
@azasypkin
azasypkin committed Nov 19, 2023
1 parent 02c2c2e commit 83f48c0
Show file tree
Hide file tree
Showing 12 changed files with 221 additions and 8 deletions.
2 changes: 2 additions & 0 deletions src/scheduler.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,7 @@ mod tests {
schedule: Some("1 2 3 4 5 6 2030".to_string()),
delay: Default::default(),
scripts: Default::default(),
headers: Default::default(),
enable_notifications: true,
},
},
Expand All @@ -232,6 +233,7 @@ mod tests {
schedule: Some("1 2 3 4 5 6 2030".to_string()),
delay: Default::default(),
scripts: Default::default(),
headers: Default::default(),
enable_notifications: true,
},
},
Expand Down
26 changes: 26 additions & 0 deletions src/scheduler/scheduler_jobs/web_page_trackers_fetch_job.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,7 @@ mod tests {
schedule: Some("0 0 * * * *".to_string()),
delay: Default::default(),
scripts: Default::default(),
headers: Default::default(),
enable_notifications: true,
},
},
Expand Down Expand Up @@ -525,6 +526,7 @@ mod tests {
schedule: Some("0 0 * * * *".to_string()),
delay: Default::default(),
scripts: Default::default(),
headers: Default::default(),
enable_notifications: true,
},
},
Expand Down Expand Up @@ -610,6 +612,11 @@ mod tests {
.into_iter()
.collect(),
),
headers: Some(
[("cookie".to_string(), "my-cookie".to_string())]
.into_iter()
.collect(),
),
enable_notifications: true,
},
user_id: user.id,
Expand Down Expand Up @@ -660,6 +667,11 @@ mod tests {
.set_scripts(WebScraperResourcesRequestScripts {
resource_filter_map: Some("return resource;"),
})
.set_headers(
&[("cookie".to_string(), "my-cookie".to_string())]
.into_iter()
.collect(),
)
.set_delay(Duration::from_millis(2000)),
)
.unwrap(),
Expand Down Expand Up @@ -778,6 +790,11 @@ mod tests {
.into_iter()
.collect(),
),
headers: Some(
[("cookie".to_string(), "my-cookie".to_string())]
.into_iter()
.collect(),
),
enable_notifications: false,
},
user_id: user.id,
Expand Down Expand Up @@ -815,6 +832,11 @@ mod tests {
.set_scripts(WebScraperContentRequestScripts {
extract_content: Some("return document.body.innerText;"),
})
.set_headers(
&[("cookie".to_string(), "my-cookie".to_string())]
.into_iter()
.collect(),
)
.set_delay(Duration::from_millis(2000)),
)
.unwrap(),
Expand Down Expand Up @@ -918,6 +940,7 @@ mod tests {
schedule: Some(tracker_schedule),
delay: Duration::from_secs(2),
scripts: Default::default(),
headers: Default::default(),
enable_notifications: true,
},
user_id: user.id,
Expand Down Expand Up @@ -1099,6 +1122,7 @@ mod tests {
schedule: Some(tracker_schedule),
delay: Duration::from_secs(2),
scripts: Default::default(),
headers: Default::default(),
enable_notifications: true,
},
user_id: user.id,
Expand Down Expand Up @@ -1265,6 +1289,7 @@ mod tests {
schedule: Some(tracker_schedule),
delay: Duration::from_secs(2),
scripts: Default::default(),
headers: Default::default(),
enable_notifications: true,
},
user_id: user.id,
Expand Down Expand Up @@ -1430,6 +1455,7 @@ mod tests {
schedule: Some(tracker_schedule),
delay: Duration::from_secs(2),
scripts: Default::default(),
headers: Default::default(),
enable_notifications: true,
},
user_id: user.id,
Expand Down
7 changes: 7 additions & 0 deletions src/scheduler/scheduler_jobs/web_page_trackers_schedule_job.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,7 @@ mod tests {
enable_notifications: true,
schedule: Some("1 2 3 4 5 6 2030".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand All @@ -279,6 +280,7 @@ mod tests {
enable_notifications: true,
schedule: Some("1 2 3 4 5 6 2035".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand All @@ -296,6 +298,7 @@ mod tests {
enable_notifications: true,
schedule: Some("1 2 3 4 5 6 2040".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand Down Expand Up @@ -437,6 +440,7 @@ mod tests {
enable_notifications: true,
schedule: None,
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand All @@ -455,6 +459,7 @@ mod tests {
enable_notifications: true,
schedule: None,
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand Down Expand Up @@ -534,6 +539,7 @@ mod tests {
enable_notifications: true,
schedule: Some("1 2 3 4 5 6 2030".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand All @@ -551,6 +557,7 @@ mod tests {
enable_notifications: true,
schedule: Some("1 2 3 4 5 6 2030".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand Down
5 changes: 5 additions & 0 deletions src/scheduler/scheduler_jobs/web_page_trackers_trigger_job.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ mod tests {
schedule: Some("0 0 * * * *".to_string()),
delay: Default::default(),
scripts: Default::default(),
headers: Default::default(),
enable_notifications: true,
},
},
Expand Down Expand Up @@ -308,6 +309,7 @@ mod tests {
schedule: Some("0 0 * * * *".to_string()),
delay: Default::default(),
scripts: Default::default(),
headers: Default::default(),
enable_notifications: true,
},
},
Expand Down Expand Up @@ -390,6 +392,7 @@ mod tests {
schedule: Some("1 0 * * * *".to_string()),
delay: Default::default(),
scripts: Default::default(),
headers: Default::default(),
enable_notifications: true,
},
},
Expand Down Expand Up @@ -455,6 +458,7 @@ mod tests {
schedule: None,
delay: Default::default(),
scripts: Default::default(),
headers: Default::default(),
enable_notifications: true,
},
},
Expand Down Expand Up @@ -516,6 +520,7 @@ mod tests {
schedule: Some("0 0 * * * *".to_string()),
delay: Default::default(),
scripts: Default::default(),
headers: Default::default(),
enable_notifications: true,
},
},
Expand Down
17 changes: 17 additions & 0 deletions src/utils/web_scraping.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ pub mod tests {
delay: Duration::from_millis(2000),
schedule: None,
scripts: Default::default(),
headers: Default::default(),
enable_notifications: true,
},
created_at: OffsetDateTime::from_unix_timestamp(946720800)?,
Expand Down Expand Up @@ -234,6 +235,7 @@ pub mod tests {
enable_notifications: true,
schedule: Some("0 0 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand Down Expand Up @@ -304,6 +306,7 @@ pub mod tests {
enable_notifications: true,
schedule: Some("0 0 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand Down Expand Up @@ -365,6 +368,7 @@ pub mod tests {
enable_notifications: true,
schedule: Some("0 0 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
}
}))),
)
Expand Down Expand Up @@ -414,6 +418,7 @@ pub mod tests {
enable_notifications: true,
schedule: Some("0 0 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
}
}))),
)
Expand Down Expand Up @@ -462,6 +467,7 @@ pub mod tests {
enable_notifications: true,
schedule: Some("0 0 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand All @@ -483,6 +489,7 @@ pub mod tests {
enable_notifications: false,
schedule: Some("0 1 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
}
}))),
)
Expand All @@ -509,6 +516,7 @@ pub mod tests {
enable_notifications: false,
schedule: Some("0 1 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
created_at: tracker.created_at,
meta: None
Expand Down Expand Up @@ -537,6 +545,7 @@ pub mod tests {
enable_notifications: true,
schedule: Some("0 0 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand All @@ -558,6 +567,7 @@ pub mod tests {
enable_notifications: false,
schedule: Some("0 1 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
}
}))),
)
Expand All @@ -584,6 +594,7 @@ pub mod tests {
enable_notifications: false,
schedule: Some("0 1 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
created_at: tracker.created_at,
meta: None
Expand Down Expand Up @@ -612,6 +623,7 @@ pub mod tests {
enable_notifications: true,
schedule: Some("0 0 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand Down Expand Up @@ -658,6 +670,7 @@ pub mod tests {
enable_notifications: true,
schedule: Some("0 0 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand Down Expand Up @@ -705,6 +718,7 @@ pub mod tests {
enable_notifications: true,
schedule: Some("0 0 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand All @@ -722,6 +736,7 @@ pub mod tests {
enable_notifications: true,
schedule: Some("0 0 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand Down Expand Up @@ -851,6 +866,7 @@ pub mod tests {
enable_notifications: true,
schedule: Some("0 0 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand All @@ -867,6 +883,7 @@ pub mod tests {
enable_notifications: true,
schedule: Some("0 0 * * * *".to_string()),
scripts: Default::default(),
headers: Default::default(),
},
},
)
Expand Down
Loading

0 comments on commit 83f48c0

Please sign in to comment.