From 33c39f792af0f77a26ac64b645e09185fd098c30 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 4 Oct 2023 16:49:18 +0200 Subject: [PATCH 1/2] load builder toolchain from database instead of environment --- src/config.rs | 3 --- src/docbuilder/rustwide_builder.rs | 36 ++++++++++++++++++------------ src/utils/mod.rs | 1 + 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/config.rs b/src/config.rs index 69a41eff5..e44dae942 100644 --- a/src/config.rs +++ b/src/config.rs @@ -99,7 +99,6 @@ pub struct Config { pub(crate) rustwide_workspace: PathBuf, pub(crate) inside_docker: bool, pub(crate) docker_image: Option, - pub(crate) toolchain: String, pub(crate) build_cpu_limit: Option, pub(crate) build_default_memory_limit: Option, pub(crate) include_default_targets: bool, @@ -113,7 +112,6 @@ impl Config { ("CRATESFYI_DATABASE_URL", "DOCSRS_DATABASE_URL"), ("CRATESFYI_GITHUB_ACCESSTOKEN", "DOCSRS_GITHUB_ACCESSTOKEN"), ("CRATESFYI_RUSTWIDE_WORKSPACE", "DOCSRS_RUSTWIDE_WORKSPACE"), - ("CRATESFYI_TOOLCHAIN", "DOCSRS_TOOLCHAIN"), ("DOCS_RS_DOCKER", "DOCSRS_DOCKER"), ("DOCS_RS_LOCAL_DOCKER_IMAGE", "DOCSRS_DOCKER_IMAGE"), ("DOCS_RS_BULID_CPU_LIMIT", "DOCSRS_BULID_CPU_LIMIT"), @@ -202,7 +200,6 @@ impl Config { inside_docker: env("DOCSRS_DOCKER", false)?, docker_image: maybe_env("DOCSRS_LOCAL_DOCKER_IMAGE")? .or(maybe_env("DOCSRS_DOCKER_IMAGE")?), - toolchain: env("DOCSRS_TOOLCHAIN", "nightly".to_string())?, build_cpu_limit: maybe_env("DOCSRS_BUILD_CPU_LIMIT")?, build_default_memory_limit: maybe_env("DOCSRS_BUILD_DEFAULT_MEMORY_LIMIT")?, include_default_targets: env("DOCSRS_INCLUDE_DEFAULT_TARGETS", true)?, diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 6d2980e5d..36036dfde 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -8,8 +8,8 @@ use crate::error::Result; use crate::repositories::RepositoryStatsUpdater; use crate::storage::{rustdoc_archive_path, source_archive_path}; use crate::utils::{ - copy_dir_all, parse_rustc_version, queue_builder, report_error, set_config, CargoMetadata, - ConfigName, + copy_dir_all, get_config, parse_rustc_version, queue_builder, report_error, set_config, + CargoMetadata, ConfigName, }; use crate::RUSTDOC_STATIC_STORAGE_PREFIX; use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage}; @@ -33,6 +33,21 @@ const COMPONENTS: &[&str] = &["llvm-tools-preview", "rustc-dev", "rustfmt"]; const DUMMY_CRATE_NAME: &str = "empty-library"; const DUMMY_CRATE_VERSION: &str = "1.0.0"; +fn get_configured_toolchain(conn: &mut Client) -> Result { + let name: String = get_config(conn, ConfigName::Toolchain)?.unwrap_or_else(|| "nightly".into()); + + // If the toolchain is all hex, assume it references an artifact from + // CI, for instance an `@bors try` build. + let re = Regex::new(r"^[a-fA-F0-9]+$").unwrap(); + if re.is_match(&name) { + debug!("using CI build {}", &name); + Ok(Toolchain::ci(&name, false)) + } else { + debug!("using toolchain {}", &name); + Ok(Toolchain::dist(&name)) + } +} + pub enum PackageKind<'a> { Local(&'a Path), CratesIo, @@ -75,22 +90,13 @@ impl RustwideBuilder { .purge_all_build_dirs() .map_err(FailureError::compat)?; - // If the toolchain is all hex, assume it references an artifact from - // CI, for instance an `@bors try` build. - let re = Regex::new(r"^[a-fA-F0-9]+$").unwrap(); - let toolchain = if re.is_match(&config.toolchain) { - debug!("using CI build {}", &config.toolchain); - Toolchain::ci(&config.toolchain, false) - } else { - debug!("using toolchain {}", &config.toolchain); - Toolchain::dist(&config.toolchain) - }; + let pool = context.pool()?; Ok(RustwideBuilder { workspace, - toolchain, + toolchain: get_configured_toolchain(&mut *pool.get()?)?, config, - db: context.pool()?, + db: pool, storage: context.storage()?, metrics: context.instance_metrics()?, index: context.index()?, @@ -119,6 +125,8 @@ impl RustwideBuilder { } pub fn update_toolchain(&mut self) -> Result { + self.toolchain = get_configured_toolchain(&mut *self.db.get()?)?; + // For CI builds, a lot of the normal update_toolchain things don't apply. // CI builds are only for one platform (https://forge.rust-lang.org/infra/docs/rustc-ci.html#try-builds) // so we only try installing for the current platform. If that's not a match, diff --git a/src/utils/mod.rs b/src/utils/mod.rs index c58d05aba..c62ab4423 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -56,6 +56,7 @@ pub enum ConfigName { RustcVersion, LastSeenIndexReference, QueueLocked, + Toolchain, } pub fn set_config( From a73d1c5e7a84a5c5bc358a3c7e517b98e0a41890 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 4 Oct 2023 17:25:56 +0200 Subject: [PATCH 2/2] add build set-toolchain subcommand --- src/bin/cratesfyi.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 8bead0151..ea3dca362 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -12,7 +12,7 @@ use docs_rs::db::{self, add_path_into_database, Overrides, Pool, PoolClient}; use docs_rs::repositories::RepositoryStatsUpdater; use docs_rs::utils::{ get_config, get_crate_pattern_and_priority, list_crate_priorities, queue_builder, - remove_crate_priority, set_crate_priority, ConfigName, + remove_crate_priority, set_config, set_crate_priority, ConfigName, }; use docs_rs::{ start_background_metrics_webserver, start_web_server, BuildQueue, Config, Context, Index, @@ -383,6 +383,10 @@ enum BuildSubcommand { /// Adds essential files for the installed version of rustc AddEssentialFiles, + SetToolchain { + toolchain_name: String, + }, + /// Locks the daemon, preventing it from building new crates Lock, @@ -459,6 +463,15 @@ impl BuildSubcommand { .context("failed to add essential files")?; } + Self::SetToolchain { toolchain_name } => { + let mut conn = ctx + .pool()? + .get() + .context("failed to get a database connection")?; + set_config(&mut conn, ConfigName::Toolchain, toolchain_name) + .context("failed to set toolchain in database")?; + } + Self::Lock => build_queue.lock().context("Failed to lock")?, Self::Unlock => build_queue.unlock().context("Failed to unlock")?, }