diff --git a/Cargo.toml b/Cargo.toml index fc7e07bc445..4d53fc5bea6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,8 +32,8 @@ pio = ["embuild/pio"] paste = "1" [build-dependencies] -embuild = { version = "0.28.4", features = ["glob"] } +embuild = { version = "0.29.0", features = ["glob"] } anyhow = "1" -strum = { version = "0.23", features = ["derive"], optional = true } +strum = { version = "0.24", features = ["derive"], optional = true } regex = "1.5" bindgen = "0.59" diff --git a/README.md b/README.md index 8297f5c507d..110bb4c4fd7 100644 --- a/README.md +++ b/README.md @@ -106,27 +106,40 @@ The following environment variables are used by the build script: - `ESP_IDF_TOOLS_INSTALL_DIR`: - The location where the ESP-IDF framework tooling is assumed to be/will be installed. - The framework tooling is either PlatformIO (when the `pio` builder is used), or the ESP-IDF native toolset (when the `native` builder is used). + The location where the ESP-IDF framework tooling is assumed to be/will be installed. The + framework tooling is either PlatformIO (when the `pio` builder is used), or the ESP-IDF + native toolset (when the `native` builder is used). - This variable can take one of the following values: + This variable can take one of the following values: - `workspace` (default) - the tooling will be installed/used in `/.embuild/platformio` for `pio`, and `/.embuild/espressif` for the `native` builder; - `out` - the tooling will be installed/used inside the crate's build output directory, and will be deleted when `cargo clean` is invoked; - `global` - the tooling will be installed/used in its standard directory (`~/.platformio` for PlatformIO, and `~./espressif` for the native ESP-IDF toolset); - `custom:` - the tooling will be installed/used in the directory specified by ``. If this directory is a relative location, it is assumed to be - relative to the crate's workspace dir. - - **ATTENTION**: Please be extra careful with the `custom:` setting when switching from `pio` to `native` and the other way around, because - the builder will install the tooling in `` without using any additional `platformio` or `espressif` subdirectories, so if you are not careful, you might end up with - both PlatformIO, as well as the ESP-IDF native tooling intermingled together in a single folder. - - - Note that both builders (`native` and `pio`) clone the ESP-IDF GIT repository *inside* the tooling directory as well. This restriction might be lifted soon for the `native` builder, whereas the user would be able to point the build to a custom ESP-IDF repository location. + relative to the crate's workspace dir; + - `fromenv` - use the build framework from the environment + - *native* builder: use activated esp-idf environment (see esp-idf docs + [unix](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/get-started/linux-macos-setup.html#step-4-set-up-the-environment-variables) + / + [windows](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/get-started/windows-setup.html#using-the-command-prompt)) + - *pio* builder: use `platformio` from the environment (i.e. `$PATH`) + + and error if this is not possible. + + **ATTENTION**: Please be extra careful with the `custom:` setting when switching from `pio` to `native` and the other way around, because + the builder will install the tooling in `` without using any additional `platformio` or `espressif` subdirectories, so if you are not careful, you might end up with + both PlatformIO, as well as the ESP-IDF native tooling intermingled together in a single folder. + + + Note that both builders (`native` and `pio`) clone the ESP-IDF GIT repository *inside* the tooling directory as well. This restriction might be lifted soon for the `native` builder, whereas the user would be able to point the build to a custom ESP-IDF repository location. + +- `IDF_PATH` (*native* builder only): + A path to a user-provided local clone of the [`esp-idf`](https://github.com/espressif/esp-idf), + that will be used instead of the one downloaded by the build script. - `ESP_IDF_VERSION` (*native* builder only): - The version used for the `esp-idf` can be one of the following: + The version used for the `esp-idf`, can be one of the following: - `commit:`: Uses the commit `` of the `esp-idf` repository. Note that this will clone the whole `esp-idf` not just one commit. - `tag:`: Uses the tag `` of the `esp-idf` repository. diff --git a/build/common.rs b/build/common.rs index c95f7e29f65..7ec5e044d77 100644 --- a/build/common.rs +++ b/build/common.rs @@ -1,10 +1,11 @@ use std::collections::HashSet; +use std::fmt::Display; use std::iter::once; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::{env, error, fs}; -use anyhow::*; +use anyhow::{anyhow, bail, Result}; use embuild::cargo::{self, IntoWarning}; use embuild::utils::{OsStrExt, PathExt}; use embuild::{bindgen, build, kconfig}; @@ -24,9 +25,6 @@ pub const V_4_3_2_PATCHES: &[&str] = &[ "patches/pthread_destructor_fix.diff", ]; -#[allow(unused)] -pub const NO_PATCHES: &[&str] = &[]; - const TOOLS_WORKSPACE_INSTALL_DIR: &str = ".embuild"; const ALL_COMPONENTS: &[&str] = &[ @@ -207,32 +205,85 @@ pub fn list_specific_sdkconfigs( }) } -pub fn get_install_dir(builder_name: impl AsRef) -> Result> { - let location = match env::var(ESP_IDF_TOOLS_INSTALL_DIR_VAR) { - Err(env::VarError::NotPresent) => None, - var => Some(var?.to_lowercase()), - }; - - let dir = match location.as_deref() { - None | Some("workspace") => Some( - workspace_dir()? - .join(TOOLS_WORKSPACE_INSTALL_DIR) - .join(builder_name.as_ref()), - ), - Some("global") => None, - Some("out") => Some(cargo::out_dir().join(builder_name.as_ref())), - Some(custom) => { - if let Some(suffix) = custom.strip_prefix("custom:") { - Some(PathBuf::from(suffix).abspath_relative_to(&workspace_dir()?)) - } else { - bail!("Invalid installation directory format. Should be one of `global`, `workspace`, `out` or `custom:`"); +#[derive(Clone, Debug)] +pub enum InstallDir { + Global, + Workspace(PathBuf), + Out(PathBuf), + Custom(PathBuf), + FromEnv, +} + +impl InstallDir { + /// Get the install directory from the [`ESP_IDF_TOOLS_INSTALL_DIR_VAR`] env variable. + /// + /// If this env variable is unset or empty uses `default_install_dir` instead. + /// On success returns `(install_dir as InstallDir, is_default as bool)`. + pub fn from_env_or( + default_install_dir: &str, + builder_name: &str, + ) -> Result<(InstallDir, bool)> { + let location = env::var_os(ESP_IDF_TOOLS_INSTALL_DIR_VAR); + let (location, is_default) = match &location { + None => (default_install_dir, true), + Some(val) => { + let val = val.try_to_str()?.trim(); + if val.is_empty() { + (default_install_dir, true) + } else { + (val, false) + } } + }; + let install_dir = match location.to_lowercase().as_str() { + "global" => Self::Global, + "workspace" => Self::Workspace( + workspace_dir()? + .join(TOOLS_WORKSPACE_INSTALL_DIR) + .join(builder_name), + ), + "out" => Self::Out(cargo::out_dir().join(builder_name)), + "fromenv" => Self::FromEnv, + _ => Self::Custom({ + if let Some(suffix) = location.strip_prefix("custom:") { + Path::new(suffix).abspath_relative_to(&workspace_dir()?) + } else { + bail!( + "Invalid installation directory format. \ + Should be one of `global`, `workspace`, `out`, `fromenv` or `custom:`." + ); + } + }), + }; + Ok((install_dir, is_default)) + } + + pub fn is_from_env(&self) -> bool { + matches!(self, Self::FromEnv) + } + + pub fn path(&self) -> Option<&Path> { + match self { + Self::Global | Self::FromEnv => None, + Self::Workspace(ref path) => Some(path.as_ref()), + Self::Out(ref path) => Some(path.as_ref()), + Self::Custom(ref path) => Some(path.as_ref()), } - }; + } +} - Ok(dir) +impl Display for InstallDir { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Global => write!(f, "global"), + Self::Workspace(ref path) => write!(f, "workspace ({})", path.display()), + Self::Out(ref path) => write!(f, "out ({})", path.display()), + Self::Custom(ref path) => write!(f, "custom ({})", path.display()), + Self::FromEnv => write!(f, "fromenv"), + } + } } pub fn workspace_dir() -> Result { - Ok(cargo::workspace_dir().ok_or_else(|| anyhow!("Cannot fetch crate's workspace dir"))?) + cargo::workspace_dir().ok_or_else(|| anyhow!("Cannot fetch crate's workspace dir")) } diff --git a/build/native.rs b/build/native.rs index 651766fd864..4c0ba43eb15 100644 --- a/build/native.rs +++ b/build/native.rs @@ -2,26 +2,24 @@ use std::convert::TryFrom; use std::ffi::OsString; -use std::iter; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::{env, fs}; use anyhow::{anyhow, bail, Context, Error, Result}; - -use strum::{Display, EnumString, IntoEnumIterator}; - +use embuild::cargo::IntoWarning; use embuild::cmake::file_api::codemodel::Language; use embuild::cmake::file_api::ObjKind; -use embuild::espidf::InstallOpts; +use embuild::espidf::{EspIdfOrigin, EspIdfRemote, FromEnvError}; use embuild::fs::copy_file_if_different; use embuild::utils::{OsStrExt, PathExt}; use embuild::{bindgen, build, cargo, cmake, espidf, git, kconfig, path_buf}; +use strum::{Display, EnumString, IntoEnumIterator}; use super::common::{ - self, get_install_dir, list_specific_sdkconfigs, workspace_dir, EspIdfBuildOutput, - EspIdfComponents, ESP_IDF_GLOB_VAR_PREFIX, ESP_IDF_SDKCONFIG_DEFAULTS_VAR, - ESP_IDF_SDKCONFIG_VAR, ESP_IDF_TOOLS_INSTALL_DIR_VAR, MCU_VAR, NO_PATCHES, V_4_3_2_PATCHES, + self, list_specific_sdkconfigs, workspace_dir, EspIdfBuildOutput, EspIdfComponents, InstallDir, + ESP_IDF_GLOB_VAR_PREFIX, ESP_IDF_SDKCONFIG_DEFAULTS_VAR, ESP_IDF_SDKCONFIG_VAR, + ESP_IDF_TOOLS_INSTALL_DIR_VAR, MCU_VAR, V_4_3_2_PATCHES, }; use crate::common::{SDKCONFIG_DEFAULTS_FILE, SDKCONFIG_FILE}; @@ -111,6 +109,7 @@ fn build_cargo_first() -> Result { let chip_name = chip.to_string(); let profile = common::build_profile(); + cargo::track_env_var(espidf::IDF_PATH_VAR); cargo::track_env_var(ESP_IDF_TOOLS_INSTALL_DIR_VAR); cargo::track_env_var(ESP_IDF_VERSION_VAR); cargo::track_env_var(ESP_IDF_REPOSITORY_VAR); @@ -118,61 +117,142 @@ fn build_cargo_first() -> Result { cargo::track_env_var(ESP_IDF_SDKCONFIG_VAR); cargo::track_env_var(MCU_VAR); - let cmake_tool = espidf::Tools::cmake()?; - let cmake_generator = get_cmake_generator()?; - let tools = espidf::Tools::new( - iter::once(chip.gcc_toolchain()) - .chain( - if !cfg!(target_os = "linux") && !cfg!(target_arch = "aarch64") { - chip.ulp_gcc_toolchain() - } else { - None - }, - ) - .chain(if cmake_generator == cmake::Generator::Ninja { - Some("ninja") - } else { - None - }), - ); + let make_tools = move |repo: &git::Repository, + version: &Result| + -> Result> { + eprintln!( + "Using esp-idf {} at '{}'", + espidf::EspIdfVersion::format(version), + repo.worktree().display() + ); - let install_dir = get_install_dir("espressif")?; - let idf = espidf::Installer::new(esp_idf_version()?) - .opts(InstallOpts::empty()) - .local_install_dir(install_dir.clone()) - .git_url(match env::var(ESP_IDF_REPOSITORY_VAR) { - Err(env::VarError::NotPresent) => None, - git_url => Some(git_url?), - }) - .with_tools(tools) - .with_tools(cmake_tool) - .install() - .context("Could not install esp-idf")?; - - // Apply patches, only if the patches were not previously applied. - let patch_set = match &idf.esp_idf_version { - git::Ref::Branch(b) - if b == "release/v4.4" || idf.esp_idf.get_default_branch()?.as_ref() == Some(b) => - { - NO_PATCHES + let mut tools = vec![]; + let mut subtools = vec![chip.gcc_toolchain()]; + // + // Use custom cmake for esp-idf<4.4, because we need at least cmake-3.20 + match version.as_ref().map(|v| (v.major, v.minor, v.patch)) { + Ok((major, minor, _)) if major >= 4 && minor >= 4 => subtools.push("cmake"), + _ => { + tools.push(espidf::Tools::cmake()?); + } } - git::Ref::Branch(b) if b == "release/v4.3" => V_4_3_2_PATCHES, - git::Ref::Tag(t) if t.starts_with("v4.4") => NO_PATCHES, - git::Ref::Tag(t) if t == "v4.3.2" => V_4_3_2_PATCHES, - _ => { - cargo::print_warning(format_args!( - "`esp-idf` version ({:?}) not officially supported by `esp-idf-sys`. \ - Supported versions are 'master', 'release/v4.4', 'release/v4.3', 'v4.4(.X)', 'v4.3.2'.", - &idf.esp_idf_version, - )); - &[] + + if cmake_generator == cmake::Generator::Ninja { + subtools.push("ninja") } + if !cfg!(target_os = "linux") && !cfg!(target_arch = "aarch64") { + subtools.extend(chip.ulp_gcc_toolchain()); + } + tools.push(espidf::Tools::new(subtools)); + + Ok(tools) }; - if !patch_set.is_empty() { - idf.esp_idf - .apply_once(patch_set.iter().map(|p| manifest_dir.join(p)))?; + + // Get the install dir from the $ESP_IDF_TOOLS_INSTALL_DIR, if unset use + // "workspace" and allow esp-idf from the environment. + let (install_dir, allow_from_env) = InstallDir::from_env_or("workspace", "espressif")?; + // EspIdf must come from the environment if $ESP_IDF_TOOLS_INSTALL_DIR == "fromenv". + let require_from_env = install_dir.is_from_env(); + let maybe_from_env = require_from_env || allow_from_env; + + let install = |esp_idf_origin: EspIdfOrigin| -> Result { + let (custom_url, custom_version) = esp_idf_remote_parts()?; + match &esp_idf_origin { + EspIdfOrigin::Custom(repo) => { + eprintln!( + "Using custom user-supplied esp-idf repository at '{}' (detected from env variable `{}`)", + repo.worktree().display(), + espidf::IDF_PATH_VAR + ); + if let Some(custom_url) = custom_url { + cargo::print_warning(format_args!( + "Ignoring configuration setting `{ESP_IDF_REPOSITORY_VAR}=\"{custom_url}\"`: \ + custom esp-idf repository detected via ${}", + espidf::IDF_PATH_VAR + )); + } + if let Some(custom_version) = custom_version { + cargo::print_warning(format_args!( + "Ignoring configuration setting `{ESP_IDF_VERSION_VAR}` ({custom_version}): \ + custom esp-idf repository detected via ${}", + espidf::IDF_PATH_VAR + )); + } + } + EspIdfOrigin::Managed(remote) => { + eprintln!("Using managed esp-idf repository: {remote:?}"); + } + }; + + espidf::Installer::new(esp_idf_origin) + .install_dir(install_dir.path().map(Into::into)) + .with_tools(make_tools) + .install() + .context("Could not install esp-idf") + }; + + let idf = match (espidf::EspIdf::try_from_env(), maybe_from_env) { + (Ok(idf), true) => { + eprintln!( + "Using activated esp-idf {} environment at '{}'", + espidf::EspIdfVersion::format(&idf.version), + idf.repository.worktree().display() + ); + + idf + }, + (Ok(idf), false) => { + cargo::print_warning(format_args!( + "Ignoring activated esp-idf environment: ${ESP_IDF_TOOLS_INSTALL_DIR_VAR} != {}", InstallDir::FromEnv + )); + install(EspIdfOrigin::Custom(idf.repository))? + }, + (Err(FromEnvError::NotActivated { source: err, .. }), true) | + (Err(FromEnvError::NoRepo(err)), true) if require_from_env => { + return Err(err.context( + format!("activated esp-idf environment not found but required by ${ESP_IDF_TOOLS_INSTALL_DIR_VAR} == {install_dir}") + )) + } + (Err(FromEnvError::NoRepo(_)), _) => { + let (repo_url, git_ref) = esp_idf_remote_parts()?; + let git_ref = git_ref.unwrap_or_else(|| espidf::parse_esp_idf_git_ref(DEFAULT_ESP_IDF_VERSION)); + + install(EspIdfOrigin::Managed(EspIdfRemote { + git_ref, + repo_url + }))? + }, + (Err(FromEnvError::NotActivated { esp_idf_repo, .. }), _) => { + install(EspIdfOrigin::Custom(esp_idf_repo))? + } + }; + + // Apply patches, only if the patches were not previously applied and if the esp-idf repo is managed. + if idf.is_managed_espidf { + let patch_set = match idf.version.map(|v| (v.major, v.minor, v.patch)) { + // master branch + _ if idf.repository.get_default_branch()? == idf.repository.get_branch_name()? => &[], + Ok((4, 4, _)) => &[], + Ok((4, 3, patch)) if patch >= 2 => V_4_3_2_PATCHES, + Ok((major, minor, patch)) => { + cargo::print_warning(format_args!( + "esp-idf version ({major}.{minor}.{patch}) not officially supported by `esp-idf-sys`. \ + Supported versions are 'master', 'release/v4.4', 'release/v4.3', 'v4.4(.X)', 'v4.3.2'.", + )); + &[] + } + Err(err) => { + err.context("could not determine patch-set for esp-idf repository") + .into_warning(); + &[] + } + }; + if !patch_set.is_empty() { + idf.repository + .apply_once(patch_set.iter().map(|p| manifest_dir.join(p)))?; + } } env::set_var("PATH", &idf.exported_path); @@ -263,7 +343,7 @@ fn build_cargo_first() -> Result { })?; let cmake_toolchain_file = path_buf![ - &idf.esp_idf.worktree(), + &idf.repository.worktree(), "tools", "cmake", chip.cmake_toolchain_file() @@ -279,7 +359,7 @@ fn build_cargo_first() -> Result { cmake::cmake(), "-P", extractor_script.as_ref().as_os_str(); - env=("IDF_PATH", &idf.esp_idf.worktree().as_os_str()))?; + env=("IDF_PATH", &idf.repository.worktree().as_os_str()))?; let mut vars = cmake::process_script_variables_extractor_output(output)?; ( @@ -312,12 +392,12 @@ fn build_cargo_first() -> Result { .asmflag(asm_flags) .cflag(c_flags) .cxxflag(cxx_flags) - .env("IDF_PATH", &idf.esp_idf.worktree()) + .env("IDF_PATH", &idf.repository.worktree()) .env("PATH", &idf.exported_path) .env("SDKCONFIG_DEFAULTS", defaults_files) .env("IDF_TARGET", &chip_name); - if let Some(install_dir) = install_dir { + if let Some(install_dir) = install_dir.path() { cmake_config.env("IDF_TOOLS_PATH", install_dir); } @@ -349,7 +429,7 @@ fn build_cargo_first() -> Result { // Save information about the esp-idf build to the out dir so that it can be // easily retrieved by tools that need it. espidf::EspIdfBuildInfo { - esp_idf_dir: idf.esp_idf.worktree().to_owned(), + esp_idf_dir: idf.repository.worktree().to_owned(), exported_path_var: idf.exported_path.try_to_str()?.to_owned(), venv_python: idf.venv_python, build_dir: cmake_build_dir.clone(), @@ -378,18 +458,26 @@ fn build_cargo_first() -> Result { .with_context(|| anyhow!("Failed to read '{:?}'", sdkconfig_json))?, ), env_path: Some(idf.exported_path.try_to_str()?.to_owned()), - esp_idf: idf.esp_idf.worktree().to_owned(), + esp_idf: idf.repository.worktree().to_owned(), }; Ok(build_output) } -fn esp_idf_version() -> Result { - let version = match env::var(ESP_IDF_VERSION_VAR) { - Err(env::VarError::NotPresent) => DEFAULT_ESP_IDF_VERSION.to_owned(), - v => v?, +fn esp_idf_remote_parts() -> Result<(Option, Option)> { + let version_ref = match env::var(ESP_IDF_VERSION_VAR) { + Err(env::VarError::NotPresent) => None, + v => Some(v?.trim().to_owned()), + } + .filter(|s| !s.is_empty()) + .map(|s| espidf::parse_esp_idf_git_ref(&s)); + + let repo_url = match env::var(ESP_IDF_REPOSITORY_VAR) { + Err(env::VarError::NotPresent) => None, + git_url => Some(git_url?), }; - Ok(espidf::decode_esp_idf_version_ref(&version)) + + Ok((repo_url, version_ref)) } // Generate `sdkconfig.defaults` content based on the crate manifest (`Cargo.toml`). @@ -423,7 +511,7 @@ fn generate_sdkconfig_defaults() -> Result { fn get_cmake_generator() -> Result { let generator = match env::var(ESP_IDF_CMAKE_GENERATOR) { Err(env::VarError::NotPresent) => None, - var => Some(var?), + var => Some(var?.trim().to_lowercase()), }; let generator = match generator.as_deref() { @@ -440,7 +528,7 @@ fn get_cmake_generator() -> Result { } } Some(other) => cmake::Generator::from_str(other).map_err(|_| { - anyhow::anyhow!( + anyhow!( "Invalid CMake generator. Should be either `default`, or one of [{}]", cmake::Generator::iter() .map(|e| e.into()) diff --git a/build/pio.rs b/build/pio.rs index 307704d4993..9e44434b17d 100644 --- a/build/pio.rs +++ b/build/pio.rs @@ -30,14 +30,48 @@ pub fn build() -> Result { let workspace_dir = workspace_dir()?; let profile = build_profile(); - let install_dir = get_install_dir("platformio")?; - - if let Some(install_dir) = install_dir.as_ref() { - // Workaround an issue in embuild until it is fixed in the next version - fs::create_dir_all(install_dir)?; - } + // Get the install dir from the $ESP_IDF_TOOLS_INSTALL_DIR, if unset use + // "workspace" and allow platformio from the environment. + let (install_dir, allow_from_env) = InstallDir::from_env_or("workspace", "platformio")?; + // Pio must come from the environment if $ESP_IDF_TOOLS_INSTALL_DIR == "fromenv". + let require_from_env = install_dir.is_from_env(); + let maybe_from_env = require_from_env || allow_from_env; + + let install = |install_dir: &InstallDir| -> Result { + let install_dir = install_dir.path().map(ToOwned::to_owned); + + if let Some(install_dir) = &install_dir { + // Workaround an issue in embuild until it is fixed in the next version + fs::create_dir_all(install_dir)?; + } + + pio::Pio::install(install_dir, pio::LogLevel::Standard, false) + }; - let pio = pio::Pio::install(install_dir, pio::LogLevel::Standard, false)?; + let pio = match (pio::Pio::try_from_env(), maybe_from_env) { + (Some(pio), true) => { + eprintln!( + "Using platformio from environment at '{}'", + pio.platformio_exe.display() + ); + + pio + } + (Some(_), false) => { + cargo::print_warning(format_args!( + "Ignoring platformio in environment: ${ESP_IDF_TOOLS_INSTALL_DIR_VAR} != {}", + InstallDir::FromEnv + )); + install(&install_dir)? + } + (None, true) if require_from_env => { + bail!( + "platformio not found in environment ($PATH) \ + but required by ${ESP_IDF_TOOLS_INSTALL_DIR_VAR} == {install_dir}" + ); + } + (None, _) => install(&install_dir)?, + }; let resolution = pio::Resolver::new(pio.clone()) .params(pio::ResolutionParams { @@ -101,6 +135,7 @@ pub fn build() -> Result { let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); for patch in V_4_3_2_PATCHES { + // TODO: fix patches not applying builder.platform_package_patch(manifest_dir.join(patch), path_buf!["framework-espidf"]); }