Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

show pending CDN invalidations on queue page #1897

Merged
merged 2 commits into from
Nov 14, 2022
Merged

show pending CDN invalidations on queue page #1897

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7e74894
show pending CDN invalidations on queue page
syphar Nov 7, 2022
fbe9eab
fix typo in comment
syphar Nov 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 90 additions & 1 deletion src/cdn.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ use aws_sdk_cloudfront::{
model::{InvalidationBatch, Paths},
Client, RetryConfig,
};
use chrono::{DateTime, Utc};
use serde::Serialize;
use std::sync::{Arc, Mutex};
use strum::EnumString;
use tokio::runtime::Runtime;
Expand Down Expand Up @@ -133,14 +135,51 @@ pub(crate) fn invalidate_crate(config: &Config, cdn: &CdnBackend, name: &str) ->
Ok(())
}

#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub(crate) struct CrateInvalidation {
pub name: String,
pub created: DateTime<Utc>,
}

/// Return fake active cloudfront invalidations.
/// CloudFront invalidations can take up to 15 minutes. Until we have
/// live queries of the invalidation status we just assume it's fine
/// 20 minutes after the build.
/// TODO: should be replaced be keeping track or querying the active invalidation from CloudFront
pub(crate) fn active_crate_invalidations(
conn: &mut postgres::Client,
) -> Result<Vec<CrateInvalidation>> {
Ok(conn
.query(
r#"
SELECT
crates.name,
MIN(builds.build_time) as build_time
FROM crates
INNER JOIN releases ON crates.id = releases.crate_id
INNER JOIN builds ON releases.id = builds.rid
WHERE builds.build_time >= CURRENT_TIMESTAMP - INTERVAL '20 minutes'
GROUP BY crates.name
ORDER BY MIN(builds.build_time)"#,
&[],
)?
.iter()
.map(|row| CrateInvalidation {
name: row.get(0),
created: row.get(1),
})
.collect())
}

#[cfg(test)]
mod tests {
use super::*;
use crate::test::wrapper;
use crate::test::{wrapper, FakeBuild};

use aws_sdk_cloudfront::{Client, Config, Credentials, Region};
use aws_smithy_client::{erase::DynConnector, test_connection::TestConnection};
use aws_smithy_http::body::SdkBody;
use chrono::{Duration, Timelike};

#[test]
fn create_cloudfront() {
Expand Down Expand Up @@ -213,6 +252,56 @@ mod tests {
Config::new(&cfg)
}

#[test]
fn get_active_invalidations() {
wrapper(|env| {
let now = Utc::now().with_nanosecond(0).unwrap();
let past_deploy = now - Duration::minutes(21);
let first_running_deploy = now - Duration::minutes(10);
let second_running_deploy = now;

env.fake_release()
.name("krate_2")
.version("0.0.1")
.builds(vec![FakeBuild::default().build_time(first_running_deploy)])
.create()?;

env.fake_release()
.name("krate_2")
.version("0.0.2")
.builds(vec![FakeBuild::default().build_time(second_running_deploy)])
.create()?;

env.fake_release()
.name("krate_1")
.version("0.0.2")
.builds(vec![FakeBuild::default().build_time(second_running_deploy)])
.create()?;

env.fake_release()
.name("krate_1")
.version("0.0.3")
.builds(vec![FakeBuild::default().build_time(past_deploy)])
.create()?;

assert_eq!(
active_crate_invalidations(&mut env.db().conn())?,
vec![
CrateInvalidation {
name: "krate_2".into(),
created: first_running_deploy,
},
CrateInvalidation {
name: "krate_1".into(),
created: second_running_deploy,
}
]
);

Ok(())
})
}

#[tokio::test]
async fn invalidate_path() {
let conn = TestConnection::new(vec![(
Expand Down
15 changes: 15 additions & 0 deletions src/test/fakes.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ pub(crate) struct FakeRelease<'a> {
pub(crate) struct FakeBuild {
s3_build_log: Option<String>,
db_build_log: Option<String>,
build_time: Option<DateTime<Utc>>,
result: BuildResult,
}

Expand Down Expand Up @@ -458,6 +459,12 @@ impl FakeGithubStats {
}

impl FakeBuild {
pub(crate) fn build_time(self, build_time: impl Into<DateTime<Utc>>) -> Self {
Self {
build_time: Some(build_time.into()),
..self
}
}
pub(crate) fn rustc_version(self, rustc_version: impl Into<String>) -> Self {
Self {
result: BuildResult {
Expand Down Expand Up @@ -525,6 +532,13 @@ impl FakeBuild {
)?;
}

if let Some(build_time) = self.build_time.as_ref() {
conn.query(
"UPDATE builds SET build_time = $2 WHERE id = $1",
&[&build_id, &build_time],
)?;
}

if let Some(s3_build_log) = self.s3_build_log.as_deref() {
let path = format!("build-logs/{}/{}.txt", build_id, default_target);
storage.store_one(path, s3_build_log)?;
Expand All @@ -539,6 +553,7 @@ impl Default for FakeBuild {
Self {
s3_build_log: Some("It works!".into()),
db_build_log: None,
build_time: None,
result: BuildResult {
rustc_version: "rustc 2.0.0-nightly (000000000 1970-01-01)".into(),
docsrs_version: "docs.rs 1.0.0 (000000000 1970-01-01)".into(),
Expand Down
51 changes: 48 additions & 3 deletions src/web/releases.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

use crate::{
build_queue::QueuedCrate,
cdn::{self, CrateInvalidation},
db::{Pool, PoolClient},
impl_webpage,
utils::report_error,
Expand Down Expand Up @@ -668,6 +669,7 @@ pub fn activity_handler(req: &mut Request) -> IronResult<Response> {
struct BuildQueuePage {
description: &'static str,
queue: Vec<QueuedCrate>,
active_deployments: Vec<CrateInvalidation>,
}

impl_webpage! {
Expand All @@ -683,9 +685,12 @@ pub fn build_queue_handler(req: &mut Request) -> IronResult<Response> {
krate.priority = -krate.priority;
}

let mut conn = extension!(req, Pool).get()?;

BuildQueuePage {
description: "List of crates scheduled to build",
description: "crate documentation scheduled to build & deploy",
queue,
active_deployments: ctry!(req, cdn::active_crate_invalidations(&mut conn)),
}
.into_response(req)
}
Expand All @@ -695,7 +700,8 @@ mod tests {
use super::*;
use crate::index::api::CrateOwner;
use crate::test::{
assert_redirect, assert_redirect_unchecked, assert_success, wrapper, TestFrontend,
assert_redirect, assert_redirect_unchecked, assert_success, wrapper, FakeBuild,
TestFrontend,
};
use anyhow::Error;
use chrono::{Duration, TimeZone};
Expand Down Expand Up @@ -1326,6 +1332,40 @@ mod tests {
})
}

#[test]
fn test_deployment_queue() {
wrapper(|env| {
let web = env.frontend();

env.fake_release()
.name("krate_2")
.version("0.0.1")
.builds(vec![
FakeBuild::default().build_time(Utc::now() - Duration::minutes(10))
])
.create()?;

let empty = kuchiki::parse_html().one(web.get("/releases/queue").send()?.text()?);
assert!(empty
.select(".release > strong")
.expect("missing heading")
.any(|el| el.text_contents().contains("active CDN deployments")));

let full = kuchiki::parse_html().one(web.get("/releases/queue").send()?.text()?);
let items = full
.select(".queue-list > li")
.expect("missing list items")
.collect::<Vec<_>>();

assert_eq!(items.len(), 1);
let a = items[0].as_node().select_first("a").expect("missing link");

assert!(a.text_contents().contains("krate_2"));

Ok(())
});
}

#[test]
fn test_releases_queue() {
wrapper(|env| {
Expand All @@ -1334,10 +1374,15 @@ mod tests {

let empty = kuchiki::parse_html().one(web.get("/releases/queue").send()?.text()?);
assert!(empty
.select(".release > strong")
.select(".queue-list > strong")
.expect("missing heading")
.any(|el| el.text_contents().contains("nothing")));

assert!(!empty
.select(".release > strong")
.expect("missing heading")
.any(|el| el.text_contents().contains("active CDN deployments")));

queue.add_crate("foo", "1.0.0", 0, None)?;
queue.add_crate("bar", "0.1.0", -10, None)?;
queue.add_crate("baz", "0.0.1", 10, None)?;
Expand Down
2 changes: 1 addition & 1 deletion templates/footer.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<div class="docs-rs-footer">
<a href="/about">About docs.rs</a>
<a href="https://foundation.rust-lang.org/policies/privacy-policy/#docs.rs">Privacy policy</a>
<a href="/releases/queue">Build queue</a>
<a href="/releases/queue">Queue</a>
</div>
63 changes: 45 additions & 18 deletions templates/releases/build_queue.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,37 +1,64 @@
{%- extends "base.html" -%}
{%- import "releases/header.html" as release_macros -%}

{%- block title -%}Build Queue - Docs.rs{%- endblock title -%}
{%- block title -%}Queue - Docs.rs{%- endblock title -%}

{%- block header -%}
{{ release_macros::header(title="Build Queue", description=description, tab="queue") }}
{{ release_macros::header(title="Queue", description=description, tab="queue") }}
{%- endblock header -%}

{%- block body -%}
<div class="container">
<div class="recent-releases-container">
{%- if active_deployments %}
<div class="release">
<strong>active CDN deployments</strong>
</div>

<div class = "pure-g">
<div class="pure-u-1-2">
<ol class="queue-list">
{% for invalidation in active_deployments -%}
<li>
<a href="https://docs.rs/{{ invalidation.name }}">
{{ invalidation.name }}
</a>
</li>
{%- endfor %}
</ol>
</div>
<div class="pure-u-1-2">
<div class="about">
<p>
After the build finishes it may take up to 20 minutes for all documentation
pages to be up-to-date and available to everybody.
</p>
<p>Especially <code>/latest/</code> URLs might be affected.</p>
</div>
</div>
</div>
{%- endif %}

<div class="release">
{% set queue_length = queue | length -%}
{%- if queue_length == 0 -%}
<strong>There is nothing in the queue</strong>
{%- else -%}
<strong>Queue</strong>
{%- endif %}
<strong>Build Queue</strong>
</div>

<ol class="queue-list">
{% for crate in queue -%}
<li>
<a href="https://crates.io/crates/{{ crate.name }}">
{{ crate.name }} {{ crate.version }}
</a>
{%- if queue -%}
{% for crate in queue -%}
<li>
<a href="https://crates.io/crates/{{ crate.name }}">
{{ crate.name }} {{ crate.version }}
</a>

{% if crate.priority != 0 -%}
(priority: {{ crate.priority }})
{%- endif %}
</li>
{%- endfor %}
{% if crate.priority != 0 -%}
(priority: {{ crate.priority }})
{%- endif %}
</li>
{%- endfor %}
{%- else %}
<strong>There is nothing in the queue</strong>
{%- endif %}
</ol>
</div>
</div>
Expand Down