From ed381a3da54da80e0d2d875e40d44ec906c17e13 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 16:05:43 +0000 Subject: [PATCH 1/7] fix(env): resolve dotenv ${VAR} against the file's own values before ambient env Vendor dotenv-ng-core as crates/mise-dotenv and use it instead of dotenvy, whose substitution prefers the process environment over earlier assignments in the same file. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_012ARBMJii5DSHj4MqUWLi8q --- Cargo.lock | 25 +- Cargo.toml | 3 +- crates/mise-dotenv/Cargo.toml | 14 + crates/mise-dotenv/LICENSE | 22 + crates/mise-dotenv/README.md | 8 + crates/mise-dotenv/src/err.rs | 171 ++++ crates/mise-dotenv/src/iter.rs | 174 ++++ crates/mise-dotenv/src/key.rs | 104 +++ crates/mise-dotenv/src/lib.rs | 1071 +++++++++++++++++++++++ crates/mise-dotenv/src/parse.rs | 1354 ++++++++++++++++++++++++++++++ crates/mise-dotenv/src/render.rs | 363 ++++++++ docs/environments/index.md | 2 +- e2e/env/test_env_file_ambient | 23 + src/config/env_directive/file.rs | 85 +- src/config/mod.rs | 13 +- 15 files changed, 3343 insertions(+), 89 deletions(-) create mode 100644 crates/mise-dotenv/Cargo.toml create mode 100644 crates/mise-dotenv/LICENSE create mode 100644 crates/mise-dotenv/README.md create mode 100644 crates/mise-dotenv/src/err.rs create mode 100644 crates/mise-dotenv/src/iter.rs create mode 100644 crates/mise-dotenv/src/key.rs create mode 100644 crates/mise-dotenv/src/lib.rs create mode 100644 crates/mise-dotenv/src/parse.rs create mode 100644 crates/mise-dotenv/src/render.rs create mode 100644 e2e/env/test_env_file_ambient diff --git a/Cargo.lock b/Cargo.lock index 03819c3e623..6596fba9d3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3036,12 +3036,6 @@ dependencies = [ "litrs", ] -[[package]] -name = "dotenvy" -version = "0.15.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" - [[package]] name = "dsa" version = "0.6.3" @@ -6533,7 +6527,6 @@ dependencies = [ "dashmap 6.2.1", "demand", "diffy", - "dotenvy", "duct", "dunce", "exec", @@ -6568,6 +6561,7 @@ dependencies = [ "mise-brew-metadata", "mise-brew-relocation", "mise-cache-core", + "mise-dotenv", "mise-interactive-config", "mise-settings", "mise-util", @@ -6709,6 +6703,14 @@ dependencies = [ "url", ] +[[package]] +name = "mise-dotenv" +version = "0.1.0" +dependencies = [ + "proptest", + "temp-env", +] + [[package]] name = "mise-interactive-config" version = "2026.1.12" @@ -10678,6 +10680,15 @@ dependencies = [ "xattr", ] +[[package]] +name = "temp-env" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96374855068f47402c3121c6eed88d29cb1de8f3ab27090e273e420bdabcf050" +dependencies = [ + "parking_lot", +] + [[package]] name = "tempfile" version = "3.27.0" diff --git a/Cargo.toml b/Cargo.toml index 86d91656a95..9106ff60887 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "crates/mise-sigstore", "crates/mise-settings", "crates/mise-util", + "crates/mise-dotenv", ] [package] @@ -129,7 +130,7 @@ contracts = "0.6" dashmap = "6" demand = "2" diffy = "0.5" -dotenvy = "0.15" +mise-dotenv = { path = "crates/mise-dotenv", version = "0.1" } duct = "1" dunce = "1" expr-lang = "2" diff --git a/crates/mise-dotenv/Cargo.toml b/crates/mise-dotenv/Cargo.toml new file mode 100644 index 00000000000..ec29fb07b61 --- /dev/null +++ b/crates/mise-dotenv/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "mise-dotenv" +version = "0.1.0" +edition = "2024" +license = "MIT" +description = "dotenv parser with explicit control over variable substitution, vendored from dotenv-ng-core" +repository = "https://github.com/jdx/mise" + +[dev-dependencies] +proptest = { version = "1", default-features = false, features = ["std"] } +temp-env = "0.3" + +[lib] +doctest = false diff --git a/crates/mise-dotenv/LICENSE b/crates/mise-dotenv/LICENSE new file mode 100644 index 00000000000..ba154162bb7 --- /dev/null +++ b/crates/mise-dotenv/LICENSE @@ -0,0 +1,22 @@ +# The MIT License (MIT) + +Copyright (c) 2014 Santiago Lapresta and contributors +Copyright (c) 2026 Domen Kožar + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/crates/mise-dotenv/README.md b/crates/mise-dotenv/README.md new file mode 100644 index 00000000000..987f384b88f --- /dev/null +++ b/crates/mise-dotenv/README.md @@ -0,0 +1,8 @@ +# mise-dotenv + +A dotenv parser used by mise for `.env` files. Vendored and trimmed from +[`dotenv-ng-core`](https://github.com/cachix/dotenv-ng) 1.0.0 (itself a fork of +[`dotenvy`](https://github.com/allan2/dotenvy)), MIT licensed; see `LICENSE`. + +Unlike upstream's loader, `mise_dotenv::parse` never reads the process environment, so a file's own +assignments are not shadowed by ambient variables. Process-environment mutation was removed. diff --git a/crates/mise-dotenv/src/err.rs b/crates/mise-dotenv/src/err.rs new file mode 100644 index 00000000000..4aa5d936062 --- /dev/null +++ b/crates/mise-dotenv/src/err.rs @@ -0,0 +1,171 @@ +use std::{error, ffi::OsString, fmt, io, path::PathBuf}; + +use crate::ParseError; +use crate::iter::ParseBufError; + +#[derive(Debug)] +#[non_exhaustive] +pub enum Error { + /// The input did not conform to the supported dotenv syntax. + Parse(ParseError, Option), + /// An IO error may be encountered when reading from a file or reader. + Io(io::Error, Option), + /// The variable was not found in the environment. The `String` is the name of the variable. + NotPresent(String), + /// The variable was not valid unicode. The `String` is the name of the variable. + NotUnicode(OsString, String), + /// An environment variable name was not valid Unicode. + NotUnicodeName(OsString), + /// When a load function is called with no path or reader. + /// + /// This can occur when `EnvLoader::with_paths` or `EnvLoader::with_readers` receives an empty + /// iterator. + NoInput, +} + +impl Error { + #[must_use] + pub fn not_found(&self) -> bool { + if let Self::Io(e, _) = self { + e.kind() == io::ErrorKind::NotFound + } else { + false + } + } +} + +impl error::Error for Error { + fn source(&self) -> Option<&(dyn error::Error + 'static)> { + match self { + Self::Io(e, _) => Some(e), + Self::Parse(error, _) => Some(error), + Self::NotPresent(_) + | Self::NotUnicode(_, _) + | Self::NotUnicodeName(_) + | Self::NoInput => None, + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::Io(e, path) => { + if let Some(path) = path { + write!(f, "error reading '{}': {e}", path.to_string_lossy()) + } else { + e.fmt(f) + } + } + Self::Parse(error, Some(path)) => { + write!(f, "error parsing '{}':\n{error}", path.to_string_lossy()) + } + Self::Parse(error, None) => error.fmt(f), + Self::NotPresent(s) => write!(f, "{s} is not set"), + Self::NotUnicode(os_str, s) => { + write!(f, "{s} is not valid Unicode: {os_str:?}") + } + Self::NotUnicodeName(os_str) => { + write!( + f, + "environment variable name is not valid Unicode: {os_str:?}" + ) + } + Self::NoInput => write!(f, "no input provided"), + } + } +} + +impl From<(io::Error, PathBuf)> for Error { + fn from((e, path): (io::Error, PathBuf)) -> Self { + Self::Io(e, Some(path)) + } +} + +impl From for Error { + fn from(error: io::Error) -> Self { + Self::Io(error, None) + } +} + +impl From<(ParseBufError, Option)> for Error { + fn from((e, path): (ParseBufError, Option)) -> Self { + match e { + ParseBufError::Parse(error) => Self::Parse(error, path), + ParseBufError::Io(e) => Self::Io(e, path), + } + } +} + +#[cfg(test)] +mod tests { + use super::Error; + use crate::{EnvLoader, EnvSequence, iter::ParseBufError}; + use std::{ + error::Error as StdError, + ffi::OsString, + io::{self, Cursor}, + path::PathBuf, + }; + + fn parse_error() -> crate::ParseError { + let error = EnvLoader::with_reader(Cursor::new("BROKEN value")) + .sequence(EnvSequence::InputOnly) + .load() + .unwrap_err(); + let Error::Parse(error, _) = error else { + panic!("expected a parse error"); + }; + error + } + + #[test] + fn reports_underlying_error_sources() { + let io_error = Error::from(io::Error::other("reader failed")); + assert_eq!(io_error.source().unwrap().to_string(), "reader failed"); + + let parse_error = Error::Parse(parse_error(), None); + assert!(parse_error.source().is_some()); + + assert!(Error::NotPresent("MISSING".to_owned()).source().is_none()); + } + + #[test] + fn displays_every_context_free_error_variant() { + assert_eq!( + Error::from(io::Error::other("reader failed")).to_string(), + "reader failed" + ); + assert!( + Error::Parse(parse_error(), None) + .to_string() + .starts_with("dotenv syntax error at line 1, column 8") + ); + assert_eq!( + Error::NotPresent("MISSING".to_owned()).to_string(), + "MISSING is not set" + ); + assert_eq!( + Error::NotUnicode(OsString::from("raw-value"), "VALUE".to_owned()).to_string(), + "VALUE is not valid Unicode: \"raw-value\"" + ); + assert_eq!( + Error::NotUnicodeName(OsString::from("raw-name")).to_string(), + "environment variable name is not valid Unicode: \"raw-name\"" + ); + assert_eq!(Error::NoInput.to_string(), "no input provided"); + assert!(!Error::NoInput.not_found()); + } + + #[test] + fn converts_reader_errors_with_optional_path_context() { + let error = Error::from(( + ParseBufError::Io(io::Error::other("reader failed")), + Some(PathBuf::from("config/.env")), + )); + assert_eq!( + error.to_string(), + "error reading 'config/.env': reader failed" + ); + } +} diff --git a/crates/mise-dotenv/src/iter.rs b/crates/mise-dotenv/src/iter.rs new file mode 100644 index 00000000000..96fa8be12cc --- /dev/null +++ b/crates/mise-dotenv/src/iter.rs @@ -0,0 +1,174 @@ +use crate::{ + EnvMap, ParseError, ParseOptions, + parse::{Parser, SubstitutionTracker}, +}; +use std::{ + collections::HashMap, + io::{self, Read}, +}; + +pub struct Iter { + reader: Option, + parser: Option, + options: ParseOptions, + base_variables: HashMap, + protected_variables: HashMap, + substitution_tracker: Option, + input_overrides: bool, + read_error: Option, +} + +impl Iter { + pub(super) const fn new( + reader: B, + options: ParseOptions, + base_variables: HashMap, + protected_variables: HashMap, + substitution_tracker: Option, + input_overrides: bool, + ) -> Self { + Self { + reader: Some(reader), + parser: None, + options, + base_variables, + protected_variables, + substitution_tracker, + input_overrides, + read_error: None, + } + } + + fn initialize(&mut self) { + if let Some(mut reader) = self.reader.take() { + let mut source = String::new(); + match reader.read_to_string(&mut source) { + Ok(_) => { + self.parser = Some(Parser::new( + source, + self.options, + std::mem::take(&mut self.base_variables), + std::mem::take(&mut self.protected_variables), + self.substitution_tracker.take(), + self.input_overrides, + )); + } + Err(error) => self.read_error = Some(error), + } + } + } + + fn internal_load(mut self) -> Result { + let mut map = EnvMap::new(); + for item in &mut self { + let (key, value) = item?; + map.insert(key, value); + } + Ok(map) + } + + pub(super) fn load(self) -> Result { + self.internal_load() + } +} + +impl Iterator for Iter { + type Item = Result<(String, String), ParseBufError>; + + fn next(&mut self) -> Option { + self.initialize(); + + if let Some(error) = self.read_error.take() { + return Some(Err(ParseBufError::Io(error))); + } + + let parser = self.parser.as_mut()?; + let result = parser.next_entry()?; + Some(result.map_err(ParseBufError::Parse)) + } +} + +/// An internal error that does not yet have path context. +#[derive(Debug)] +pub enum ParseBufError { + Parse(ParseError), + Io(io::Error), +} + +impl From for ParseBufError { + fn from(error: io::Error) -> Self { + Self::Io(error) + } +} + +#[cfg(test)] +mod tests { + use super::{Iter, ParseBufError}; + use crate::ParseOptions; + use std::{collections::HashMap, io}; + + #[test] + fn iterator_strips_bom() { + let input = b"\xef\xbb\xbfkey=value\n"; + let entries: Result, _> = Iter::new( + &input[..], + ParseOptions::new(), + HashMap::new(), + HashMap::new(), + None, + true, + ) + .collect(); + assert_eq!(entries.unwrap(), [("key".to_owned(), "value".to_owned())]); + } + + #[test] + fn read_errors_are_returned_once() { + struct BrokenReader; + + impl io::Read for BrokenReader { + fn read(&mut self, _buffer: &mut [u8]) -> io::Result { + Err(io::Error::other("broken reader")) + } + } + + let mut iter = Iter::new( + BrokenReader, + ParseOptions::new(), + HashMap::new(), + HashMap::new(), + None, + true, + ); + assert!(matches!(iter.next(), Some(Err(ParseBufError::Io(_))))); + assert!(iter.next().is_none()); + } + + #[test] + fn successful_iterators_remain_exhausted() { + let mut iter = Iter::new( + &b"KEY=value\n"[..], + ParseOptions::new(), + HashMap::new(), + HashMap::new(), + None, + true, + ); + + assert_eq!( + iter.next().unwrap().unwrap(), + ("KEY".to_owned(), "value".to_owned()) + ); + assert!(iter.next().is_none()); + assert!(iter.next().is_none()); + } + + #[test] + fn io_errors_convert_to_internal_buffer_errors() { + let error = ParseBufError::from(io::Error::other("reader failed")); + let ParseBufError::Io(error) = error else { + panic!("expected an I/O error"); + }; + assert_eq!(error.to_string(), "reader failed"); + } +} diff --git a/crates/mise-dotenv/src/key.rs b/crates/mise-dotenv/src/key.rs new file mode 100644 index 00000000000..12ba938c2e4 --- /dev/null +++ b/crates/mise-dotenv/src/key.rs @@ -0,0 +1,104 @@ +use std::collections::HashMap; + +pub fn get<'a, V>(map: &'a HashMap, key: &str) -> Option<&'a V> { + if let Some(value) = map.get(key) { + return Some(value); + } + + #[cfg(windows)] + { + map.iter() + .find_map(|(candidate, value)| equivalent(candidate, key).then_some(value)) + } + #[cfg(not(windows))] + { + None + } +} + +pub fn contains(map: &HashMap, key: &str) -> bool { + get(map, key).is_some() +} + +pub fn insert(map: &mut HashMap, key: String, value: V) -> Option { + #[cfg(windows)] + let previous = if map.contains_key(&key) { + None + } else { + let existing = map + .keys() + .find(|candidate| equivalent(candidate, &key)) + .cloned(); + existing.and_then(|existing| map.remove(&existing)) + }; + + #[cfg(not(windows))] + let previous = None; + + map.insert(key, value).or(previous) +} + +pub fn remove(map: &mut HashMap, key: &str) -> Option { + if map.contains_key(key) { + return map.remove(key); + } + + #[cfg(windows)] + { + let existing = map + .keys() + .find(|candidate| equivalent(candidate, key)) + .cloned(); + existing.and_then(|existing| map.remove(&existing)) + } + #[cfg(not(windows))] + { + None + } +} + +#[cfg(windows)] +fn equivalent(left: &str, right: &str) -> bool { + const CSTR_EQUAL: i32 = 2; + const TRUE: i32 = 1; + + #[link(name = "kernel32")] + extern "system" { + fn CompareStringOrdinal( + left: *const u16, + left_len: i32, + right: *const u16, + right_len: i32, + ignore_case: i32, + ) -> i32; + } + + if left == right { + return true; + } + + let left: Vec<_> = left.encode_utf16().collect(); + let right: Vec<_> = right.encode_utf16().collect(); + let (Ok(left_len), Ok(right_len)) = (i32::try_from(left.len()), i32::try_from(right.len())) + else { + return false; + }; + + // SAFETY: both pointers refer to initialized UTF-16 buffers for the explicit lengths passed + // to the Windows API, and the function does not retain them. + unsafe { + CompareStringOrdinal(left.as_ptr(), left_len, right.as_ptr(), right_len, TRUE) == CSTR_EQUAL + } +} + +#[cfg(all(test, windows))] +mod tests { + use super::equivalent; + + #[test] + fn windows_key_comparison_uses_the_os_case_table() { + assert!(equivalent("Path", "PATH")); + assert!(equivalent("\u{00e5}", "\u{00c5}")); + assert!(!equivalent("Path", "Path2")); + } +} diff --git a/crates/mise-dotenv/src/lib.rs b/crates/mise-dotenv/src/lib.rs new file mode 100644 index 00000000000..14bfcc10cdd --- /dev/null +++ b/crates/mise-dotenv/src/lib.rs @@ -0,0 +1,1071 @@ +#![deny(clippy::uninlined_format_args, clippy::wildcard_imports)] + +//! Shared dotenv parsing and loading implementation. +//! +//! This library allows for loading environment variables from an env file or a reader. +use crate::{iter::Iter, parse::SubstitutionTracker}; +use std::{ + cell::RefCell, + collections::{HashMap, HashSet}, + env::{self, VarError}, + fs::File, + io::{BufReader, Read}, + ops::{Deref, DerefMut}, + path::{Path, PathBuf}, + rc::Rc, +}; + +mod err; +mod iter; +mod key; +mod parse; +mod render; + +pub use crate::parse::{ParseError, ParseErrorKind, ParseOptions}; +pub use crate::render::{RenderError, render, render_value, render_var}; + +/// A map of environment variables. +/// +/// This is a newtype around `HashMap`. Its name-based operations follow the +/// platform's environment semantics: case-insensitive on Windows and case-sensitive elsewhere. +#[derive(Default, Clone, Debug, PartialEq, Eq)] +pub struct EnvMap(HashMap); + +impl Deref for EnvMap { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for EnvMap { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl FromIterator<(String, String)> for EnvMap { + fn from_iter>(iter: I) -> Self { + let mut map = Self::new(); + map.extend(iter); + map + } +} + +impl Extend<(String, String)> for EnvMap { + fn extend>(&mut self, iter: I) { + for (key, value) in iter { + self.insert(key, value); + } + } +} + +impl IntoIterator for EnvMap { + type Item = (String, String); + type IntoIter = std::collections::hash_map::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl EnvMap { + #[must_use] + pub fn new() -> Self { + Self(HashMap::new()) + } + + /// Returns the value for an environment variable name. + /// + /// Names are case-insensitive on Windows and case-sensitive on other platforms, matching the + /// operating system's process-environment behavior. + #[must_use] + pub fn get(&self, key: &str) -> Option<&String> { + key::get(&self.0, key) + } + + /// Returns whether the map contains an environment variable name. + /// + /// Names are case-insensitive on Windows and case-sensitive on other platforms. + #[must_use] + pub fn contains_key(&self, key: &str) -> bool { + key::contains(&self.0, key) + } + + /// Inserts an environment variable, returning the previous value if present. + /// + /// On Windows, an existing key that differs only by case is replaced instead of creating a + /// second entry. + pub fn insert(&mut self, key: String, value: String) -> Option { + key::insert(&mut self.0, key, value) + } + + /// Removes an environment variable, returning its value if present. + /// + /// Names are case-insensitive on Windows and case-sensitive on other platforms. + pub fn remove(&mut self, key: &str) -> Option { + key::remove(&mut self.0, key) + } + + pub fn var(&self, key: &str) -> Result { + self.get(key) + .cloned() + .ok_or_else(|| Error::NotPresent(key.to_owned())) + } +} + +pub use crate::err::Error; + +fn environment_variables() -> Result { + env::vars_os() + .map(|(name, value)| { + let name = name.into_string().map_err(Error::NotUnicodeName)?; + let value = value + .into_string() + .map_err(|value| Error::NotUnicode(value, name.clone()))?; + Ok((name, value)) + }) + .collect() +} + +/// Fetches the environment variable `key` from the current process. +/// +/// This is `std::env::var` but with an error type of [`Error`]. +/// [`Error`] uses `NotPresent(String)` instead of `NotPresent`, reporting the name of the missing key. +/// +/// # Errors +/// +/// This function will return an error if the environment variable isn't set. +/// +/// This function may return an error if the environment variable's name contains +/// the equal sign character (`=`) or the NUL character. +/// +/// This function will return an error if the environment variable's value is +/// not valid Unicode. +/// +/// # Examples +/// +/// ``` +/// let key = "HOME"; +/// match dotenv_ng_core::var(key) { +/// Ok(val) => println!("{key}: {val:?}"), +/// Err(e) => println!("couldn't interpret {key}: {e}"), +/// } +/// ``` +pub fn var(key: &str) -> Result { + env::var(key).map_err(|e| match e { + VarError::NotPresent => Error::NotPresent(key.to_owned()), + VarError::NotUnicode(os_str) => Error::NotUnicode(os_str, key.to_owned()), + }) +} + +/// Parses dotenv `content`, returning the assignments in file order. +/// +/// With `substitution` enabled, `$NAME` and `${NAME}` resolve against earlier assignments in +/// `content` first and then against `substitutions`. The process environment is never consulted, +/// so ambient variables cannot shadow a value defined in the file itself. +pub fn parse( + content: &str, + substitution: bool, + substitutions: I, +) -> Result, Error> +where + I: IntoIterator, +{ + let mut protected = HashMap::new(); + for (key, value) in substitutions { + key::insert(&mut protected, key, value); + } + let iter = Iter::new( + content.as_bytes(), + ParseOptions::new().substitution(substitution), + HashMap::new(), + protected, + None, + true, + ); + iter.collect::, _>>() + .map_err(|error| Error::from((error, None::))) +} + +/// The sequence in which to load environment variables. +/// +/// Values in the latter override values in the former. +#[derive(Default, Debug, PartialEq, Eq, Clone, Copy)] +pub enum EnvSequence { + /// Inherit the existing environment without loading from input. + EnvOnly, + /// Inherit the existing environment, and then load from input, overriding existing values. + EnvThenInput, + /// Load from input only. + InputOnly, + /// Load from input and then inherit the existing environment. Values in the existing environment are not overwritten. + #[default] + InputThenEnv, +} + +enum Input<'a> { + Path(PathBuf), + Reader { + reader: Box, + path: Option, + }, +} + +pub struct EnvLoader<'a> { + inputs: Vec>, + sequence: EnvSequence, + parse_options: ParseOptions, + substitutions: EnvMap, + substitution_tracker: Option, + ignore_missing: bool, +} + +impl Default for EnvLoader<'_> { + fn default() -> Self { + Self::new() + } +} + +impl<'a> EnvLoader<'a> { + fn empty() -> Self { + Self { + inputs: Vec::new(), + sequence: EnvSequence::default(), + parse_options: ParseOptions::default(), + substitutions: EnvMap::default(), + substitution_tracker: None, + ignore_missing: false, + } + } + + #[must_use] + /// Creates a new `EnvLoader` with the path set to `./.env` in the current directory. + pub fn new() -> Self { + Self::with_path("./.env") + } + + /// Creates a new `EnvLoader` with the path as input. + /// + /// This operation is infallible. IO is deferred until `load` is called. + pub fn with_path>(path: P) -> Self { + Self { + inputs: vec![Input::Path(path.as_ref().to_owned())], + ..Self::empty() + } + } + + /// Creates a loader for multiple paths. + /// + /// Files are parsed in the supplied order. Later files override earlier files, and values + /// from earlier files are available to substitutions in later files. All files are parsed + /// successfully before [`load`](Self::load) returns. + pub fn with_paths(paths: I) -> Self + where + I: IntoIterator, + P: AsRef, + { + Self { + inputs: paths + .into_iter() + .map(|path| Input::Path(path.as_ref().to_owned())) + .collect(), + ..Self::empty() + } + } + + /// Creates a new `EnvLoader` with the reader as input. + /// + /// This operation is infallible. IO is deferred until `load` is called. + pub fn with_reader(rdr: R) -> Self { + Self { + inputs: vec![Input::Reader { + reader: Box::new(rdr), + path: None, + }], + ..Self::empty() + } + } + + /// Creates a loader for multiple readers. + /// + /// Readers have the same ordering and substitution behavior as paths supplied to + /// [`with_paths`](Self::with_paths). + pub fn with_readers(readers: I) -> Self + where + I: IntoIterator, + R: Read + 'a, + { + Self { + inputs: readers + .into_iter() + .map(|reader| Input::Reader { + reader: Box::new(reader) as Box, + path: None, + }) + .collect(), + ..Self::empty() + } + } + + /// Sets the path to the specified path. + /// + /// This is useful when constructing with a reader, but still desiring a path to be used in the error message context. + /// + /// If a reader exists and a path is specified, loading will be done using the reader. + #[must_use] + pub fn path>(mut self, path: P) -> Self { + let path = path.as_ref().to_owned(); + if let [Input::Reader { path: context, .. }] = self.inputs.as_mut_slice() { + *context = Some(path); + } else { + self.inputs = vec![Input::Path(path)]; + } + self + } + + /// Controls whether a missing path is an error. + /// + /// Missing paths are errors by default. When `required` is false, missing paths are skipped; + /// parse errors and all other I/O errors are still returned. + #[must_use] + pub const fn required(mut self, required: bool) -> Self { + self.ignore_missing = !required; + self + } + + /// Sets the sequence in which to load environment variables. + #[must_use] + pub const fn sequence(mut self, sequence: EnvSequence) -> Self { + self.sequence = sequence; + self + } + + /// Enables or disables variable substitution while parsing input. + /// + /// Substitution is disabled by default, preserving dollar signs and + /// substitution expressions literally. + #[must_use] + pub const fn substitution(mut self, enabled: bool) -> Self { + self.parse_options = self.parse_options.substitution(enabled); + self + } + + /// Enables or disables physical newlines inside quoted values. + /// + /// Multiline quoted values are enabled by default. + #[must_use] + pub const fn multiline(mut self, enabled: bool) -> Self { + self.parse_options = self.parse_options.multiline(enabled); + self + } + + /// Sets all parser options at once. + #[must_use] + pub const fn parse_options(mut self, options: ParseOptions) -> Self { + self.parse_options = options; + self + } + + /// Adds values that are visible to substitution without modifying the process environment. + /// + /// Call [`substitution(true)`](Self::substitution) to enable interpolation. Explicit + /// substitutions take precedence over process environment values. They are used only while + /// parsing and are not included in the returned map unless the input defines them. + #[must_use] + pub fn substitutions(mut self, values: I) -> Self + where + I: IntoIterator, + K: Into, + V: Into, + { + self.substitutions.extend( + values + .into_iter() + .map(|(key, value)| (key.into(), value.into())), + ); + self + } + + fn load_input(self, environment: &EnvMap) -> Result { + if self.inputs.is_empty() { + return Err(Error::NoInput); + } + + let protects_environment = match self.sequence { + EnvSequence::EnvThenInput | EnvSequence::InputThenEnv => true, + EnvSequence::EnvOnly | EnvSequence::InputOnly => false, + }; + let mut protected_variables = if protects_environment { + environment.0.clone() + } else { + HashMap::new() + }; + for (key, value) in self.substitutions { + key::insert(&mut protected_variables, key, value); + } + + let mut accumulated = EnvMap::new(); + for input in self.inputs { + let opened: Option<(Box, _)> = match input { + Input::Path(path) => match File::open(&path) { + Ok(file) => Some((Box::new(file), Some(path))), + Err(error) + if self.ignore_missing && error.kind() == std::io::ErrorKind::NotFound => + { + None + } + Err(error) => return Err((error, path).into()), + }, + Input::Reader { reader, path } => Some((reader, path)), + }; + if let Some((reader, path)) = opened { + let iter = Iter::new( + BufReader::new(reader), + self.parse_options, + accumulated.0.clone(), + protected_variables.clone(), + self.substitution_tracker.clone(), + !matches!(self.sequence, EnvSequence::InputThenEnv), + ); + let parsed = iter + .load() + .map_err(|error| crate::Error::from((error, path)))?; + accumulated.extend(parsed); + } + } + Ok(accumulated) + } + + /// Loads environment variables into a hash map. + /// + /// This is the primary method for loading environment variables. + pub fn load(self) -> Result { + let environment = if matches!(self.sequence, EnvSequence::InputOnly) { + EnvMap::new() + } else { + environment_variables()? + }; + match self.sequence { + EnvSequence::EnvOnly => Ok(environment), + EnvSequence::EnvThenInput => { + let mut existing = environment.clone(); + let input = self.load_input(&environment)?; + existing.extend(input); + Ok(existing) + } + EnvSequence::InputOnly => self.load_input(&environment), + EnvSequence::InputThenEnv => { + let mut input = self.load_input(&environment)?; + input.extend(environment); + Ok(input) + } + } + } + + /// Loads the environment and reports every variable consulted during substitution. + /// + /// This is an implementation hook for the compile-time macros, which must make all + /// environment reads visible to Cargo's incremental compilation dependency tracking. + #[doc(hidden)] + pub fn load_with_substitution_dependencies( + mut self, + ) -> Result<(EnvMap, HashSet), crate::Error> { + let tracker: SubstitutionTracker = Rc::new(RefCell::new(HashSet::new())); + self.substitution_tracker = Some(Rc::clone(&tracker)); + let map = self.load()?; + let dependencies = tracker.borrow().clone(); + Ok((map, dependencies)) + } +} + +#[cfg(test)] +mod tests { + use crate::{EnvLoader, EnvSequence, Error, Input, ParseErrorKind, ParseOptions, parse}; + use std::path::Path; + use std::{ + env, fs, + io::Cursor, + time::{SystemTime, UNIX_EPOCH}, + }; + + #[cfg(unix)] + use std::{ffi::OsString, os::unix::ffi::OsStringExt}; + + #[cfg(windows)] + #[test] + fn windows_environment_names_follow_case_insensitive_precedence() { + use std::io::Cursor; + + const NAME: &str = "DOTENV_NG_WINDOWS_CASE"; + temp_env::with_var("Dotenv_Ng_Windows_Case", Some("environment"), || { + let inherited = EnvLoader::with_reader(Cursor::new( + "dotenv_ng_windows_case=input\nRESULT=$DOTENV_NG_WINDOWS_CASE", + )) + .sequence(EnvSequence::InputThenEnv) + .substitution(true) + .load() + .unwrap(); + assert_eq!(inherited.var(NAME).unwrap(), "environment"); + assert_eq!(inherited.var("result").unwrap(), "environment"); + assert_eq!( + inherited + .keys() + .filter(|key| key.eq_ignore_ascii_case(NAME)) + .count(), + 1 + ); + + let overridden = EnvLoader::with_reader(Cursor::new( + "dotenv_ng_windows_case=input\nRESULT=$DOTENV_NG_WINDOWS_CASE", + )) + .sequence(EnvSequence::EnvThenInput) + .substitution(true) + .load() + .unwrap(); + assert_eq!(overridden.var(NAME).unwrap(), "input"); + assert_eq!(overridden.var("result").unwrap(), "input"); + assert_eq!( + overridden + .keys() + .filter(|key| key.eq_ignore_ascii_case(NAME)) + .count(), + 1 + ); + }); + } + + #[cfg(windows)] + #[test] + fn windows_input_and_explicit_substitutions_are_case_insensitive() { + use std::io::Cursor; + + let mut map = EnvLoader::with_reader(Cursor::new( + "dotenv_ng_mixed=first\nDOTENV_NG_MIXED=second\nRESULT=$dotenv_ng_injected", + )) + .sequence(EnvSequence::InputOnly) + .substitution(true) + .substitutions([("Dotenv_Ng_Injected", "injected")]) + .load() + .unwrap(); + + assert_eq!(map.var("dotenv_ng_mixed").unwrap(), "second"); + assert_eq!(map.var("result").unwrap(), "injected"); + assert_eq!( + map.insert("Dotenv_Ng_Mixed".to_owned(), "third".to_owned()), + Some("second".to_owned()) + ); + assert_eq!(map.var("DOTENV_NG_MIXED").unwrap(), "third"); + assert_eq!( + map.keys() + .filter(|key| key.eq_ignore_ascii_case("DOTENV_NG_MIXED")) + .count(), + 1 + ); + } + + #[cfg(unix)] + #[test] + fn non_unicode_environment_value_returns_an_error() { + let value = OsString::from_vec(vec![0xff]); + + temp_env::with_var("DOTENV_NG_NON_UNICODE", Some(value.clone()), || { + let error = EnvLoader::with_reader(Cursor::new("KEY=value")) + .load() + .unwrap_err(); + + assert!(matches!( + error, + Error::NotUnicode(actual, ref name) + if actual == value && name == "DOTENV_NG_NON_UNICODE" + )); + }); + } + + #[cfg(unix)] + #[test] + fn non_unicode_environment_name_returns_an_error() { + let name = OsString::from_vec(b"DOTENV_NG_NON_UNICODE_\xff".to_vec()); + + temp_env::with_var(name.clone(), Some("value"), || { + let error = EnvLoader::with_reader(Cursor::new("KEY=value")) + .load() + .unwrap_err(); + + assert!(matches!(error, Error::NotUnicodeName(actual) if actual == name)); + }); + } + + #[cfg(unix)] + #[test] + fn input_only_does_not_inspect_non_unicode_environment_values() { + let value = OsString::from_vec(vec![0xff]); + + temp_env::with_var("DOTENV_NG_NON_UNICODE", Some(value), || { + let map = EnvLoader::with_reader(Cursor::new("KEY=value")) + .sequence(EnvSequence::InputOnly) + .load() + .unwrap(); + + assert_eq!(map["KEY"], "value"); + }); + } + + #[test] + fn free_var_reports_present_and_missing_values() { + temp_env::with_vars( + [ + ("DOTENV_NG_PRESENT", Some("value")), + ("DOTENV_NG_MISSING", None), + ], + || { + assert_eq!(crate::var("DOTENV_NG_PRESENT").unwrap(), "value"); + assert!(matches!( + crate::var("DOTENV_NG_MISSING"), + Err(Error::NotPresent(name)) if name == "DOTENV_NG_MISSING" + )); + }, + ); + } + + #[cfg(unix)] + #[test] + fn free_var_reports_non_unicode_values() { + let value = OsString::from_vec(vec![0xff]); + temp_env::with_var("DOTENV_NG_NON_UNICODE_VAR", Some(value.clone()), || { + assert!(matches!( + crate::var("DOTENV_NG_NON_UNICODE_VAR"), + Err(Error::NotUnicode(actual, name)) + if actual == value && name == "DOTENV_NG_NON_UNICODE_VAR" + )); + }); + } + + #[test] + fn test_substitution() -> Result<(), crate::Error> { + temp_env::with_vars([("KEY", Some("value")), ("KEY1", Some("value1"))], || { + let subs = [ + "$ZZZ", "$KEY", "$KEY1", "${KEY}1", "$KEY_U", "${KEY_U}", "\\$KEY", + ]; + + let common_string = subs.join(">>"); + let s = format!( + r#" + KEY1=new_value1 + KEY_U=$KEY+valueU + + STRONG_QUOTES='{common_string}' + WEAK_QUOTES="{common_string}" + NO_QUOTES={common_string} + "#, + ); + let env_map = EnvLoader::with_reader(Cursor::new(s)) + .sequence(EnvSequence::InputThenEnv) + .substitution(true) + .load()?; + + assert_eq!(env_map.var("KEY")?, "value"); + assert_eq!(env_map.var("KEY1")?, "value1"); + assert_eq!(env_map.var("KEY_U")?, "value+valueU"); + assert_eq!(env_map.var("STRONG_QUOTES")?, common_string); + let expanded = [ + "", + "value", + "value1", + "value1", + "value+valueU", + "value+valueU", + "$KEY", + ] + .join(">>"); + assert_eq!(env_map.var("WEAK_QUOTES")?, expanded); + assert_eq!(env_map.var("NO_QUOTES")?, expanded); + Ok(()) + }) + } + + #[test] + fn test_multiline() -> Result<(), crate::Error> { + let value = "-----BEGIN PRIVATE KEY-----\n-----END PRIVATE KEY-----\\n\\\"QUOTED\\\""; + let weak = "-----BEGIN PRIVATE KEY-----\n-----END PRIVATE KEY-----\n\"QUOTED\""; + + let s = format!( + r#" + KEY=my\ cool\ value + KEY3="awesome \"stuff\" + more + on other + lines" + KEY4='hello \'world\' + good morning' + WEAK="{value}" + STRONG='{value}' + "# + ); + + let env_map = EnvLoader::with_reader(Cursor::new(s)) + .sequence(EnvSequence::InputOnly) + .load()?; + assert_eq!(env_map.var("KEY")?, "my cool value"); + assert_eq!( + env_map.var("KEY3")?, + r#"awesome "stuff" + more + on other + lines"# + ); + assert_eq!( + env_map.var("KEY4")?, + "hello 'world' + good morning" + ); + assert_eq!(env_map.var("WEAK")?, weak); + assert_eq!(env_map.var("STRONG")?, value); + Ok(()) + } + + #[test] + fn test_multiline_comment() -> Result<(), crate::Error> { + let s = r#" +# Start of env file +# Comment line with single ' quote +# Comment line with double " quote + # Comment line with double " quote and starts with a space +TESTKEY1=test_val # 1 '" comment +TESTKEY2=test_val_with_#_hash # 2 '" comment +TESTKEY3="test_val quoted with # hash" # 3 '" comment +TESTKEY4="Line 1 +# Line 2 +Line 3" # 4 Multiline "' comment +TESTKEY5="Line 4 +# Line 5 +Line 6 +" # 5 Multiline "' comment +# End of env file +"#; + + let env_map = EnvLoader::with_reader(Cursor::new(s)) + .sequence(EnvSequence::InputOnly) + .load()?; + assert_eq!(env_map.var("TESTKEY1")?, "test_val"); + assert_eq!(env_map.var("TESTKEY2")?, "test_val_with_#_hash"); + assert_eq!(env_map.var("TESTKEY3")?, "test_val quoted with # hash"); + assert_eq!( + env_map.var("TESTKEY4")?, + "Line 1 +# Line 2 +Line 3" + ); + assert_eq!( + env_map.var("TESTKEY5")?, + "Line 4 +# Line 5 +Line 6 +" + ); + Ok(()) + } + + #[test] + fn test_non_modify() -> Result<(), crate::Error> { + temp_env::with_var("SRC", Some("env"), || { + let s = "SRC=envfile\nFOO=bar"; + let env_map = EnvLoader::with_reader(Cursor::new(s)) + .sequence(EnvSequence::EnvThenInput) + .load()?; + assert_eq!("envfile", env_map.var("SRC")?); + assert_eq!("bar", env_map.var("FOO")?); + + let env_map = EnvLoader::with_reader(Cursor::new(s)) + .sequence(EnvSequence::InputThenEnv) + .load()?; + assert_eq!("env", env_map.var("SRC")?); + Ok(()) + }) + } + + #[test] + fn input_only_substitution_does_not_read_the_process_environment() { + temp_env::with_var("DOTENV_NG_EXTERNAL", Some("from-environment"), || { + let input = + "LOCAL=from-input\nLOCAL_RESULT=$LOCAL\nEXTERNAL_RESULT=$DOTENV_NG_EXTERNAL"; + let map = EnvLoader::with_reader(Cursor::new(input)) + .sequence(EnvSequence::InputOnly) + .substitution(true) + .load() + .unwrap(); + + assert_eq!(map["LOCAL_RESULT"], "from-input"); + assert_eq!(map["EXTERNAL_RESULT"], ""); + }); + } + + #[test] + fn accepts_explicit_substitution_values_without_environment_mutation() { + temp_env::with_var_unset("APP_VAR", || { + let input = "RESULT=${APP_VAR}-from-file"; + let map = EnvLoader::with_reader(Cursor::new(input)) + .sequence(EnvSequence::InputOnly) + .substitution(true) + .substitutions([("APP_VAR".to_owned(), "provided".to_owned())]) + .load() + .unwrap(); + + assert_eq!(map["RESULT"], "provided-from-file"); + assert!(env::var_os("APP_VAR").is_none()); + }); + } + + #[test] + fn multiple_inputs_share_substitutions_and_later_inputs_win() { + temp_env::with_vars( + [ + ("DOTENV_NG_MULTI_A", None), + ("DOTENV_NG_MULTI_SOURCE", Some("environment")), + ], + || { + let readers = [ + Cursor::new("DOTENV_NG_MULTI_A=base\nDOTENV_NG_MULTI_SOURCE=first\n"), + Cursor::new( + "DOTENV_NG_MULTI_A=local\nFROM_A=$DOTENV_NG_MULTI_A\n\ + DOTENV_NG_MULTI_SOURCE=second\nFROM_SOURCE=$DOTENV_NG_MULTI_SOURCE\n", + ), + ]; + let map = EnvLoader::with_readers(readers) + .sequence(EnvSequence::EnvThenInput) + .substitution(true) + .load() + .unwrap(); + + assert_eq!(map["DOTENV_NG_MULTI_A"], "local"); + assert_eq!(map["FROM_A"], "local"); + assert_eq!(map["DOTENV_NG_MULTI_SOURCE"], "second"); + assert_eq!(map["FROM_SOURCE"], "second"); + }, + ); + } + + #[test] + fn environment_stays_authoritative_across_multiple_inputs() { + temp_env::with_var("DOTENV_NG_MULTI_SOURCE", Some("environment"), || { + let readers = [ + Cursor::new("DOTENV_NG_MULTI_SOURCE=first\n"), + Cursor::new("FROM_SOURCE=$DOTENV_NG_MULTI_SOURCE\n"), + ]; + let map = EnvLoader::with_readers(readers) + .sequence(EnvSequence::InputThenEnv) + .substitution(true) + .load() + .unwrap(); + + assert_eq!(map["DOTENV_NG_MULTI_SOURCE"], "environment"); + assert_eq!(map["FROM_SOURCE"], "environment"); + }); + } + + #[test] + fn optional_missing_paths_are_skipped_before_later_files() { + let suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let existing = env::temp_dir().join(format!( + "dotenv-ng-core-{}-{suffix}.env", + std::process::id() + )); + let missing = existing.with_extension("missing"); + fs::write(&existing, "LOADED_FROM_PATH=yes\n").unwrap(); + + let map = EnvLoader::with_paths([&missing, &existing]) + .required(false) + .sequence(EnvSequence::InputOnly) + .load() + .unwrap(); + fs::remove_file(existing).unwrap(); + + assert_eq!(map["LOADED_FROM_PATH"], "yes"); + } + + #[test] + fn default_loader_matches_new_loader() { + for loader in [EnvLoader::default(), EnvLoader::new()] { + let [Input::Path(path)] = loader.inputs.as_slice() else { + panic!("the default loader should contain exactly one path"); + }; + assert_eq!(path, Path::new("./.env")); + } + } + + #[test] + fn empty_input_collections_are_rejected() { + let error = EnvLoader::with_readers(Vec::>::new()) + .sequence(EnvSequence::InputOnly) + .load() + .unwrap_err(); + assert!(matches!(error, Error::NoInput)); + + let error = EnvLoader::with_paths(Vec::<&str>::new()) + .sequence(EnvSequence::InputOnly) + .load() + .unwrap_err(); + assert!(matches!(error, Error::NoInput)); + } + + #[test] + fn path_replaces_file_input_and_contextualizes_reader_input() { + let file_error = EnvLoader::with_path("old.env") + .path("new.env") + .sequence(EnvSequence::InputOnly) + .load() + .unwrap_err(); + assert!(file_error.to_string().contains("new.env")); + assert!(!file_error.to_string().contains("old.env")); + + let reader_error = EnvLoader::with_reader(Cursor::new("BROKEN value")) + .path("reader.env") + .sequence(EnvSequence::InputOnly) + .load() + .unwrap_err(); + assert!(reader_error.to_string().contains("reader.env")); + } + + #[test] + fn loader_builder_options_control_parsing() { + let multiline_error = EnvLoader::with_reader(Cursor::new("VALUE=\"first\nsecond\"")) + .sequence(EnvSequence::InputOnly) + .multiline(false) + .load() + .unwrap_err(); + assert!(matches!( + multiline_error, + Error::Parse(ref error, _) if error.kind() == &ParseErrorKind::MultilineDisabled + )); + + let options = ParseOptions::new().substitution(true).multiline(false); + let map = EnvLoader::with_reader(Cursor::new("BASE=value\nRESULT=$BASE")) + .sequence(EnvSequence::InputOnly) + .parse_options(options) + .load() + .unwrap(); + assert_eq!(map["RESULT"], "value"); + } + + #[test] + fn missing_file_errors_retain_their_path_and_kind() { + let error = EnvLoader::with_path("definitely/missing/required.env") + .sequence(EnvSequence::InputOnly) + .load() + .unwrap_err(); + assert!(error.not_found()); + assert!( + error + .to_string() + .contains("definitely/missing/required.env") + ); + } + + #[test] + fn reports_variables_consulted_during_substitution() { + let (map, dependencies) = EnvLoader::with_reader(Cursor::new( + "DIRECT=$EXTERNAL\nBRACED=${ANOTHER}\nLITERAL='$IGNORED'\n", + )) + .sequence(EnvSequence::InputOnly) + .substitution(true) + .substitutions([("EXTERNAL", "value")]) + .load_with_substitution_dependencies() + .unwrap(); + + assert_eq!(map["DIRECT"], "value"); + assert_eq!(map["BRACED"], ""); + assert_eq!(map["LITERAL"], "$IGNORED"); + assert_eq!(dependencies.len(), 2); + assert!(dependencies.contains("EXTERNAL")); + assert!(dependencies.contains("ANOTHER")); + } + + #[test] + fn loader_preserves_bcrypt_style_values_by_default() { + // Regression for https://github.com/cachix/secretspec/issues/73: bcrypt hashes contain + // dollar-prefixed fragments that must not be treated as variable references. + let secret = "foo:$2a$10$TWoviNHS27HJMw1PKe4tBeIMlms6tWdYS9hKoHANKCQhluDlEt/gu,\ + bar:$2a$10$labXlt9fBRMjJu.gOUabjebLVBKGB/xZOFpEn/esCln56USXHMHQW"; + let input = format!("VALUE=\"{secret}\""); + let map = EnvLoader::with_reader(Cursor::new(input)) + .sequence(EnvSequence::InputOnly) + .load() + .unwrap(); + assert_eq!(map["VALUE"], secret); + } + + #[test] + fn parse_errors_include_path_line_column_and_source() { + let error = EnvLoader::with_reader(Cursor::new("VALID=yes\nBROKEN value\n")) + .path("config/.env") + .sequence(EnvSequence::InputOnly) + .load() + .unwrap_err(); + + let Error::Parse(parse_error, Some(path)) = &error else { + panic!("expected a path-aware parse error, got {error:?}"); + }; + assert_eq!(path, Path::new("config/.env")); + assert_eq!(parse_error.line(), 2); + assert_eq!(parse_error.column(), 8); + assert_eq!(parse_error.kind(), &ParseErrorKind::MissingEquals); + assert_eq!(parse_error.source_line(), "BROKEN value"); + assert!(error.to_string().contains("2 | BROKEN value")); + } + + #[test] + fn parse_prefers_own_assignments_over_substitutions_and_keeps_order() { + let parsed = parse( + "VAR_B=\"test_${VAR_A}\"\nVAR_A=test\nVAR_C=\"c_${VAR_A}_${OUTER}\"\n", + true, + [ + ("VAR_A".to_owned(), "base".to_owned()), + ("OUTER".to_owned(), "outer".to_owned()), + ], + ) + .unwrap(); + assert_eq!( + parsed, + [ + ("VAR_B".to_owned(), "test_base".to_owned()), + ("VAR_A".to_owned(), "test".to_owned()), + ("VAR_C".to_owned(), "c_test_outer".to_owned()), + ] + ); + } + + #[test] + fn parse_in_file_reference_beats_substitution_defined_earlier() { + let parsed = parse( + "VAR_A=test\nVAR_B=\"test_${VAR_A}\"\n", + true, + [("VAR_A".to_owned(), "base".to_owned())], + ) + .unwrap(); + assert_eq!(parsed[1], ("VAR_B".to_owned(), "test_test".to_owned())); + } + + #[test] + fn parse_ignores_the_process_environment() { + temp_env::with_var("DOTENV_NG_PARSE_AMBIENT", Some("ambient"), || { + let parsed = parse("A=${DOTENV_NG_PARSE_AMBIENT}\n", true, []).unwrap(); + assert_eq!(parsed[0].1, ""); + }); + } + + #[test] + fn environment_only_load_ignores_input() { + temp_env::with_var("DOTENV_NG_ENV_ONLY", Some("environment"), || { + let map = EnvLoader::with_reader(Cursor::new("BROKEN input")) + .sequence(EnvSequence::EnvOnly) + .load() + .unwrap(); + + assert_eq!(map["DOTENV_NG_ENV_ONLY"], "environment"); + }); + } +} diff --git a/crates/mise-dotenv/src/parse.rs b/crates/mise-dotenv/src/parse.rs new file mode 100644 index 00000000000..439e14b245e --- /dev/null +++ b/crates/mise-dotenv/src/parse.rs @@ -0,0 +1,1354 @@ +use std::{ + cell::RefCell, + collections::{HashMap, HashSet}, + error, fmt, + rc::Rc, +}; + +pub type SubstitutionTracker = Rc>>; + +/// Options controlling how dotenv values are parsed. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ParseOptions { + substitution: bool, + multiline: bool, +} + +impl ParseOptions { + /// Creates options using the default dotenv syntax. + /// + /// Variable substitution is disabled by default so dollar signs are + /// preserved literally. Multiline quoted values are enabled. + #[must_use] + pub const fn new() -> Self { + Self { + substitution: false, + multiline: true, + } + } + + /// Enables or disables `$NAME` and `${NAME}` substitution. + #[must_use] + pub const fn substitution(mut self, enabled: bool) -> Self { + self.substitution = enabled; + self + } + + /// Enables or disables physical newlines inside quoted values. + #[must_use] + pub const fn multiline(mut self, enabled: bool) -> Self { + self.multiline = enabled; + self + } + + /// Returns whether variable substitution is enabled. + #[must_use] + pub const fn substitution_enabled(self) -> bool { + self.substitution + } + + /// Returns whether quoted values may span physical lines. + #[must_use] + pub const fn multiline_enabled(self) -> bool { + self.multiline + } +} + +impl Default for ParseOptions { + fn default() -> Self { + Self::new() + } +} + +/// The reason a dotenv document could not be parsed. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum ParseErrorKind { + /// An assignment did not contain a key. + MissingKey, + /// A key was not followed by `=`. + MissingEquals, + /// A key contains a character that cannot be represented in an environment variable. + InvalidKeyCharacter(char), + /// A value contains a character that cannot be represented in an environment variable. + InvalidValueCharacter(char), + /// Non-comment text followed a quoted value. + TrailingCharacters, + /// A quoted value was not closed. + UnterminatedQuote(char), + /// A `${...}` expression was not closed. + UnterminatedSubstitution, + /// A `${...}` expression used unsupported syntax. + InvalidSubstitution, + /// A required variable substitution failed. + RequiredVariable { + /// The missing or empty variable. + name: String, + /// The optional message supplied in the substitution expression. + message: String, + }, + /// A quoted value crossed a line while multiline values were disabled. + MultilineDisabled, +} + +impl fmt::Display for ParseErrorKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MissingKey => f.write_str("expected an environment variable name"), + Self::MissingEquals => f.write_str("expected `=` after the environment variable name"), + Self::InvalidKeyCharacter(character) => { + write!( + f, + "environment variable name contains invalid character {character:?}" + ) + } + Self::InvalidValueCharacter(character) => { + write!( + f, + "environment variable value contains invalid character {character:?}" + ) + } + Self::TrailingCharacters => { + f.write_str("expected a comment or end of line after the quoted value") + } + Self::UnterminatedQuote(quote) => write!(f, "unterminated {quote:?} quoted value"), + Self::UnterminatedSubstitution => f.write_str("unterminated `${...}` substitution"), + Self::InvalidSubstitution => f.write_str("invalid variable substitution"), + Self::RequiredVariable { name, message } if message.is_empty() => { + write!(f, "required variable `{name}` is not set") + } + Self::RequiredVariable { name, message } => { + write!(f, "required variable `{name}` is not set: {message}") + } + Self::MultilineDisabled => f.write_str("multiline quoted values are disabled"), + } + } +} + +/// A source-aware dotenv syntax error. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ParseError { + kind: ParseErrorKind, + line: usize, + column: usize, + byte_offset: usize, + source_line: String, +} + +impl ParseError { + /// Returns the specific parsing failure. + #[must_use] + pub const fn kind(&self) -> &ParseErrorKind { + &self.kind + } + + /// Returns the one-based source line containing the error. + #[must_use] + pub const fn line(&self) -> usize { + self.line + } + + /// Returns the one-based Unicode character column containing the error. + #[must_use] + pub const fn column(&self) -> usize { + self.column + } + + /// Returns the zero-based byte offset of the error. + #[must_use] + pub const fn byte_offset(&self) -> usize { + self.byte_offset + } + + /// Returns the physical source line containing the error. + #[must_use] + pub fn source_line(&self) -> &str { + &self.source_line + } +} + +impl fmt::Display for ParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let marker_padding: String = self + .source_line + .chars() + .take(self.column.saturating_sub(1)) + .map(|character| if character == '\t' { '\t' } else { ' ' }) + .collect(); + let line_width = self.line.to_string().len(); + + writeln!( + f, + "dotenv syntax error at line {}, column {}: {}", + self.line, self.column, self.kind + )?; + writeln!(f, "{space:>line_width$} |", space = "")?; + writeln!(f, "{} | {}", self.line, self.source_line)?; + write!(f, "{space:>line_width$} | {marker_padding}^", space = "") + } +} + +impl error::Error for ParseError {} + +#[derive(Clone, Copy)] +enum ValueStyle { + Unquoted, + SingleQuoted, + DoubleQuoted, +} + +pub struct Parser { + source: String, + position: usize, + options: ParseOptions, + base_variables: HashMap, + protected_variables: HashMap, + parsed_variables: HashMap, + substitution_tracker: Option, + input_overrides: bool, + failed: bool, +} + +impl Parser { + pub(super) fn new( + mut source: String, + options: ParseOptions, + base_variables: HashMap, + protected_variables: HashMap, + substitution_tracker: Option, + input_overrides: bool, + ) -> Self { + if source.starts_with('\u{feff}') { + source.drain(..'\u{feff}'.len_utf8()); + } + + Self { + source, + position: 0, + options, + base_variables, + protected_variables, + parsed_variables: HashMap::new(), + substitution_tracker, + input_overrides, + failed: false, + } + } + + pub(super) fn next_entry(&mut self) -> Option> { + if self.failed { + return None; + } + + loop { + self.skip_horizontal_whitespace(); + + if self.at_end() { + return None; + } + if self.consume_newline() { + continue; + } + if self.peek() == Some('#') { + self.skip_comment(); + continue; + } + + let result = self.parse_assignment(); + match result { + Ok((key, value)) => { + crate::key::insert(&mut self.parsed_variables, key.clone(), value.clone()); + return Some(Ok((key, value))); + } + Err(error) => { + self.failed = true; + return Some(Err(error)); + } + } + } + } + + fn parse_assignment(&mut self) -> Result<(String, String), ParseError> { + self.consume_export_prefix(); + + let key_start = self.position; + while let Some(character) = self.peek() { + if character == '=' || is_horizontal_whitespace(character) { + break; + } + if character == '\n' || character == '\r' || character == '#' { + break; + } + if character == '\0' || character.is_control() { + return Err(self.error( + self.position, + ParseErrorKind::InvalidKeyCharacter(character), + )); + } + self.advance(); + } + + if self.position == key_start { + return Err(self.error(key_start, ParseErrorKind::MissingKey)); + } + + let key = self.source[key_start..self.position].to_owned(); + self.skip_horizontal_whitespace(); + if self.peek() != Some('=') { + return Err(self.error(self.position, ParseErrorKind::MissingEquals)); + } + self.advance(); + self.skip_horizontal_whitespace(); + + let value = match self.peek() { + None | Some('\n' | '\r' | '#') => { + if self.peek() == Some('#') { + self.skip_comment(); + } else { + self.consume_newline(); + } + String::new() + } + Some('\'') => self.parse_quoted('\'', ValueStyle::SingleQuoted)?, + Some('"') => self.parse_quoted('"', ValueStyle::DoubleQuoted)?, + Some(_) => self.parse_unquoted()?, + }; + + Ok((key, value)) + } + + fn consume_export_prefix(&mut self) { + let Some(after_export) = self.source[self.position..].strip_prefix("export") else { + return; + }; + let Some(next) = after_export.chars().next() else { + return; + }; + if !is_horizontal_whitespace(next) { + return; + } + + self.position += "export".len(); + self.skip_horizontal_whitespace(); + } + + fn parse_quoted(&mut self, quote: char, style: ValueStyle) -> Result { + let quote_position = self.position; + self.advance(); + let value_start = self.position; + let mut escaped = false; + + loop { + let Some(character) = self.peek() else { + return Err(self.error(quote_position, ParseErrorKind::UnterminatedQuote(quote))); + }; + + if character == '\n' || character == '\r' { + if !self.options.multiline { + return Err(self.error(self.position, ParseErrorKind::MultilineDisabled)); + } + escaped = false; + self.consume_newline(); + continue; + } + if character == '\0' { + return Err(self.error( + self.position, + ParseErrorKind::InvalidValueCharacter(character), + )); + } + + if character == quote && !escaped { + let value_end = self.position; + self.advance(); + let raw = &self.source[value_start..value_end]; + let value = self.expand(raw, value_start, style)?; + self.finish_quoted_value()?; + return Ok(value); + } + + if character == '\\' { + escaped = !escaped; + } else { + escaped = false; + } + self.advance(); + } + } + + fn finish_quoted_value(&mut self) -> Result<(), ParseError> { + self.skip_horizontal_whitespace(); + match self.peek() { + None => Ok(()), + Some('#') => { + self.skip_comment(); + Ok(()) + } + Some('\n' | '\r') => { + self.consume_newline(); + Ok(()) + } + Some(_) => Err(self.error(self.position, ParseErrorKind::TrailingCharacters)), + } + } + + fn parse_unquoted(&mut self) -> Result { + let value_start = self.position; + let mut quote = None; + let mut escaped = false; + let mut previous = None; + + while let Some(character) = self.peek() { + if character == '\n' || character == '\r' { + break; + } + if character == '\0' { + return Err(self.error( + self.position, + ParseErrorKind::InvalidValueCharacter(character), + )); + } + + if let Some(open_quote) = quote { + if character == open_quote && !escaped { + quote = None; + } + if character == '\\' { + escaped = !escaped; + } else { + escaped = false; + } + } else { + if character == '#' && previous.is_none_or(is_horizontal_whitespace) && !escaped { + break; + } + if matches!(character, '\'' | '"') && !escaped { + quote = Some(character); + } + escaped = character == '\\' && !escaped; + } + + previous = Some(character); + self.advance(); + } + + let value_end = self.position; + let raw = trim_unquoted_trailing_whitespace(&self.source[value_start..value_end]); + let value = self.expand(raw, value_start, ValueStyle::Unquoted)?; + + if self.peek() == Some('#') { + self.skip_comment(); + } else { + self.consume_newline(); + } + Ok(value) + } + + fn expand( + &self, + raw: &str, + source_offset: usize, + style: ValueStyle, + ) -> Result { + let mut output = String::with_capacity(raw.len()); + let mut position = 0; + + while position < raw.len() { + let character = next_char(raw, position); + + if character == '\\' { + position += character.len_utf8(); + let Some(escaped) = raw[position..].chars().next() else { + output.push('\\'); + break; + }; + position += escaped.len_utf8(); + push_escape(&mut output, style, escaped); + continue; + } + + if character == '$' + && self.options.substitution + && !matches!(style, ValueStyle::SingleQuoted) + { + position += character.len_utf8(); + self.expand_dollar(raw, &mut position, source_offset, style, &mut output)?; + continue; + } + + if character == '\r' { + position += character.len_utf8(); + if raw[position..].starts_with('\n') { + position += '\n'.len_utf8(); + } + output.push('\n'); + continue; + } + + output.push(character); + position += character.len_utf8(); + } + + Ok(output) + } + + fn expand_dollar( + &self, + raw: &str, + position: &mut usize, + source_offset: usize, + style: ValueStyle, + output: &mut String, + ) -> Result<(), ParseError> { + if raw[*position..].starts_with('$') { + *position += '$'.len_utf8(); + output.push('$'); + return Ok(()); + } + + if raw[*position..].starts_with('{') { + let expression_start = *position + '{'.len_utf8(); + let expression_end = + Self::find_substitution_end(raw, expression_start).ok_or_else(|| { + self.error( + source_offset + (*position).saturating_sub(1), + ParseErrorKind::UnterminatedSubstitution, + ) + })?; + let expression = &raw[expression_start..expression_end]; + let expanded = + self.evaluate_substitution(expression, source_offset + expression_start, style)?; + output.push_str(&expanded); + *position = expression_end + '}'.len_utf8(); + return Ok(()); + } + + let name_start = *position; + let Some(first) = raw[*position..].chars().next() else { + output.push('$'); + return Ok(()); + }; + if !is_substitution_start(first) { + output.push('$'); + return Ok(()); + } + + *position += first.len_utf8(); + while let Some(character) = raw[*position..].chars().next() { + if !is_substitution_continue(character) { + break; + } + *position += character.len_utf8(); + } + + let name = &raw[name_start..*position]; + if let Some(value) = self.variable(name) { + output.push_str(value); + } + Ok(()) + } + + fn find_substitution_end(raw: &str, mut position: usize) -> Option { + let mut depth = 1_usize; + while position < raw.len() { + if raw[position..].starts_with("${") { + depth += 1; + position += 2; + continue; + } + + let character = next_char(raw, position); + if character == '\\' { + position += character.len_utf8(); + if let Some(next) = raw[position..].chars().next() { + position += next.len_utf8(); + } + continue; + } + if character == '}' { + depth -= 1; + if depth == 0 { + return Some(position); + } + } + position += character.len_utf8(); + } + None + } + + fn evaluate_substitution( + &self, + expression: &str, + source_offset: usize, + style: ValueStyle, + ) -> Result { + if expression.is_empty() { + return Err(self.error(source_offset, ParseErrorKind::InvalidSubstitution)); + } + + // Braces can address non-shell names when an exact variable is present. + if let Some(value) = self.variable(expression) { + return Ok(value.to_owned()); + } + + let mut name_end = 0; + for (index, character) in expression.char_indices() { + let valid = if index == 0 { + is_substitution_start(character) + } else { + is_substitution_continue(character) + }; + if !valid { + break; + } + name_end = index + character.len_utf8(); + } + if name_end == 0 { + return Err(self.error(source_offset, ParseErrorKind::InvalidSubstitution)); + } + + let name = &expression[..name_end]; + let remainder = &expression[name_end..]; + if remainder.is_empty() { + return Ok(self.variable(name).unwrap_or_default().to_owned()); + } + + let (operator, word) = [":-", ":?", ":+", "-", "?", "+"] + .into_iter() + .find_map(|operator| { + remainder + .strip_prefix(operator) + .map(|word| (operator, word)) + }) + .ok_or_else(|| { + self.error( + source_offset + name_end, + ParseErrorKind::InvalidSubstitution, + ) + })?; + + let value = self.variable(name); + let is_set = value.is_some(); + let is_nonempty = value.is_some_and(|value| !value.is_empty()); + let word_offset = source_offset + name_end + operator.len(); + let expand_word = || self.expand(word, word_offset, style); + + match operator { + "-" if !is_set => expand_word(), + ":-" if !is_nonempty => expand_word(), + "+" if is_set => expand_word(), + ":+" if is_nonempty => expand_word(), + "+" | ":+" => Ok(String::new()), + "?" if !is_set => { + let message = expand_word()?; + Err(self.error( + source_offset, + ParseErrorKind::RequiredVariable { + name: name.to_owned(), + message, + }, + )) + } + ":?" if !is_nonempty => { + let message = expand_word()?; + Err(self.error( + source_offset, + ParseErrorKind::RequiredVariable { + name: name.to_owned(), + message, + }, + )) + } + "-" | ":-" | "?" | ":?" => Ok(value.unwrap_or_default().to_owned()), + _ => unreachable!("all substitution operators are handled"), + } + } + + fn variable(&self, name: &str) -> Option<&str> { + if let Some(tracker) = &self.substitution_tracker { + tracker.borrow_mut().insert(name.to_owned()); + } + + let value = if self.input_overrides { + crate::key::get(&self.parsed_variables, name) + .or_else(|| crate::key::get(&self.base_variables, name)) + .or_else(|| crate::key::get(&self.protected_variables, name)) + } else { + crate::key::get(&self.protected_variables, name) + .or_else(|| crate::key::get(&self.parsed_variables, name)) + .or_else(|| crate::key::get(&self.base_variables, name)) + }; + value.map(String::as_str) + } + + fn error(&self, byte_offset: usize, kind: ParseErrorKind) -> ParseError { + let byte_offset = byte_offset.min(self.source.len()); + let (line, line_start) = source_line_and_start(&self.source, byte_offset); + let line_end = self.source[line_start..] + .find(['\n', '\r']) + .map_or(self.source.len(), |position| line_start + position); + let column = self.source[line_start..byte_offset].chars().count() + 1; + + ParseError { + kind, + line, + column, + byte_offset, + source_line: self.source[line_start..line_end].to_owned(), + } + } + + fn skip_horizontal_whitespace(&mut self) { + while self.peek().is_some_and(is_horizontal_whitespace) { + self.advance(); + } + } + + fn skip_comment(&mut self) { + while self + .peek() + .is_some_and(|character| !matches!(character, '\n' | '\r')) + { + self.advance(); + } + self.consume_newline(); + } + + fn consume_newline(&mut self) -> bool { + match self.peek() { + Some('\r') => { + self.advance(); + if self.peek() == Some('\n') { + self.advance(); + } + true + } + Some('\n') => { + self.advance(); + true + } + _ => false, + } + } + + fn peek(&self) -> Option { + self.source[self.position..].chars().next() + } + + fn advance(&mut self) { + if let Some(character) = self.peek() { + self.position += character.len_utf8(); + } + } + + fn at_end(&self) -> bool { + self.position == self.source.len() + } +} + +fn source_line_and_start(source: &str, byte_offset: usize) -> (usize, usize) { + let bytes = source.as_bytes(); + let mut line = 1; + let mut line_start = 0; + let mut position = 0; + + while position < byte_offset { + match bytes[position] { + b'\r' => { + if position + 1 < byte_offset && bytes[position + 1] == b'\n' { + position += 1; + } + line += 1; + line_start = position + 1; + } + b'\n' => { + line += 1; + line_start = position + 1; + } + _ => {} + } + position += 1; + } + + (line, line_start) +} + +fn next_char(value: &str, position: usize) -> char { + value[position..] + .chars() + .next() + .expect("position must be on a character boundary before the end") +} + +pub fn is_horizontal_whitespace(character: char) -> bool { + character.is_whitespace() && !matches!(character, '\n' | '\r') +} + +fn trim_unquoted_trailing_whitespace(value: &str) -> &str { + let mut end = value.len(); + + while let Some((whitespace_start, character)) = value[..end].char_indices().next_back() { + if !is_horizontal_whitespace(character) { + break; + } + + let preceding_backslashes = value[..whitespace_start] + .chars() + .rev() + .take_while(|character| *character == '\\') + .count(); + if preceding_backslashes % 2 == 1 { + break; + } + end = whitespace_start; + } + + &value[..end] +} + +const fn is_substitution_start(character: char) -> bool { + character.is_ascii_alphabetic() || character == '_' +} + +const fn is_substitution_continue(character: char) -> bool { + character.is_ascii_alphanumeric() || character == '_' +} + +fn push_escape(output: &mut String, style: ValueStyle, escaped: char) { + let decoded = match style { + ValueStyle::DoubleQuoted => match escaped { + 'a' => Some('\u{0007}'), + 'b' => Some('\u{0008}'), + 'f' => Some('\u{000c}'), + 'n' => Some('\n'), + 'r' => Some('\r'), + 't' => Some('\t'), + 'v' => Some('\u{000b}'), + '\\' | '\'' | '"' | '$' => Some(escaped), + _ => None, + }, + ValueStyle::SingleQuoted => match escaped { + '\\' | '\'' => Some(escaped), + _ => None, + }, + ValueStyle::Unquoted => match escaped { + '\\' | '\'' | '"' | '$' | '#' | ' ' | '\t' => Some(escaped), + _ => None, + }, + }; + + if let Some(decoded) = decoded { + output.push(decoded); + } else { + output.push('\\'); + output.push(escaped); + } +} + +#[cfg(test)] +mod tests { + use super::{ParseErrorKind, ParseOptions, Parser}; + use std::collections::HashMap; + + fn parse(input: &str) -> Result, super::ParseError> { + parse_with_options(input, ParseOptions::new()) + } + + fn parse_with_options( + input: &str, + options: ParseOptions, + ) -> Result, super::ParseError> { + Parser::new( + input.to_owned(), + options, + HashMap::new(), + HashMap::new(), + None, + true, + ) + .collect() + } + + impl Iterator for Parser { + type Item = Result<(String, String), super::ParseError>; + + fn next(&mut self) -> Option { + self.next_entry() + } + } + + #[test] + fn parses_common_dotenv_syntax() { + let parsed = parse( + r##" +# comment +KEY=1 +KEY2 = "two words" +KEY3='literal $KEY' +KEY4=unquoted values may contain spaces +KEY5={ "json": "works", "hash": "# literal" } +EMPTY= +export EXPORTED=yes +export=also-a-key +"##, + ) + .unwrap(); + + assert_eq!( + parsed, + [ + ("KEY", "1"), + ("KEY2", "two words"), + ("KEY3", "literal $KEY"), + ("KEY4", "unquoted values may contain spaces"), + ("KEY5", r##"{ "json": "works", "hash": "# literal" }"##), + ("EMPTY", ""), + ("EXPORTED", "yes"), + ("export", "also-a-key"), + ] + .map(|(key, value)| (key.to_owned(), value.to_owned())) + ); + } + + #[test] + fn comments_only_start_outside_quotes_after_whitespace() { + let parsed = parse( + r##" +A=value#literal +B=value # comment with ' and " +C="value # literal" # comment +D='value # literal' # comment +E=before "# literal" after # comment +"##, + ) + .unwrap(); + + assert_eq!( + parsed, + [ + ("A", "value#literal"), + ("B", "value"), + ("C", "value # literal"), + ("D", "value # literal"), + ("E", "before \"# literal\" after"), + ] + .map(|(key, value)| (key.to_owned(), value.to_owned())) + ); + } + + #[test] + fn supports_multiline_quotes_and_normalizes_crlf() { + let parsed = parse("A=\"one\r\ntwo\"\r\nB='three\nfour'\n").unwrap(); + assert_eq!( + parsed, + [ + ("A".to_owned(), "one\ntwo".to_owned()), + ("B".to_owned(), "three\nfour".to_owned()), + ] + ); + } + + #[test] + fn unknown_escapes_are_preserved_for_paths_and_structured_values() { + let parsed = parse( + r#" +WINDOWS=C:\Users\me\Desktop +REGEX="\d+\s+" +ESCAPED="line one\nline two\t\\\"" +"#, + ) + .unwrap(); + assert_eq!(parsed[0].1, r"C:\Users\me\Desktop"); + assert_eq!(parsed[1].1, r"\d+\s+"); + assert_eq!(parsed[2].1, "line one\nline two\t\\\""); + } + + #[test] + fn accepts_environment_names_beyond_shell_identifiers() { + let parsed = parse("SERVICE-NAME=value\n%TEMP%=tmp\n1NUMBER=one\nключ=значение\n").unwrap(); + assert_eq!(parsed.len(), 4); + assert_eq!(parsed[0], ("SERVICE-NAME".to_owned(), "value".to_owned())); + assert_eq!(parsed[3], ("ключ".to_owned(), "значение".to_owned())); + } + + #[test] + + fn substitutes_names_with_underscores_and_shell_operators() { + let parsed = parse_with_options( + r" +FOO_BAR=test +EMPTY= +DIRECT=$FOO_BAR +BRACED=${FOO_BAR} +DEFAULT=${MISSING:-fallback} +EMPTY_DEFAULT=${EMPTY:-fallback} +SET_DEFAULT=${EMPTY-fallback} +ALTERNATE=${FOO_BAR:+yes} +LITERAL=$$FOO_BAR +", + ParseOptions::new().substitution(true), + ) + .unwrap(); + let values: HashMap<_, _> = parsed.into_iter().collect(); + + assert_eq!(values["DIRECT"], "test"); + assert_eq!(values["BRACED"], "test"); + assert_eq!(values["DEFAULT"], "fallback"); + assert_eq!(values["EMPTY_DEFAULT"], "fallback"); + assert_eq!(values["SET_DEFAULT"], ""); + assert_eq!(values["ALTERNATE"], "yes"); + assert_eq!(values["LITERAL"], "$FOO_BAR"); + } + + #[test] + fn required_substitution_is_a_source_aware_error() { + let error = parse_with_options( + "OK=yes\nVALUE=${MISSING:?configure it}\n", + ParseOptions::new().substitution(true), + ) + .unwrap_err(); + assert_eq!(error.line(), 2); + assert_eq!(error.column(), 9); + assert_eq!(error.source_line(), "VALUE=${MISSING:?configure it}"); + assert!(matches!( + error.kind(), + ParseErrorKind::RequiredVariable { name, message } + if name == "MISSING" && message == "configure it" + )); + } + + #[test] + fn substitution_is_disabled_by_default() { + assert!(!ParseOptions::new().substitution_enabled()); + assert!(!ParseOptions::default().substitution_enabled()); + assert!(ParseOptions::new().multiline_enabled()); + assert!(!ParseOptions::new().multiline(false).multiline_enabled()); + + let parsed = parse("A=value\nB=$A/${A}/$$\n").unwrap(); + assert_eq!(parsed[1].1, "$A/${A}/$$"); + } + + #[test] + fn explicit_base_variables_do_not_require_process_environment_mutation() { + let mut base = HashMap::new(); + base.insert("APP_VAR".to_owned(), "provided".to_owned()); + let parser = Parser::new( + "VALUE=${APP_VAR}-from-file".to_owned(), + ParseOptions::new().substitution(true), + base, + HashMap::new(), + None, + true, + ); + let parsed: Result, _> = parser.collect(); + assert_eq!(parsed.unwrap()[0].1, "provided-from-file"); + } + + #[test] + fn substitution_precedence_is_explicit() { + let mut base = HashMap::new(); + base.insert("VALUE".to_owned(), "environment".to_owned()); + + let input_wins: Result, _> = Parser::new( + "VALUE=input\nRESULT=$VALUE".to_owned(), + ParseOptions::new().substitution(true), + base.clone(), + HashMap::new(), + None, + true, + ) + .collect(); + assert_eq!(input_wins.unwrap()[1].1, "input"); + + let base_wins: Result, _> = Parser::new( + "VALUE=input\nRESULT=$VALUE".to_owned(), + ParseOptions::new().substitution(true), + HashMap::new(), + base, + None, + false, + ) + .collect(); + assert_eq!(base_wins.unwrap()[1].1, "environment"); + } + + #[test] + fn rejects_multiline_when_disabled() { + let mut parser = Parser::new( + "VALUE=\"first\nsecond\"".to_owned(), + ParseOptions::new().multiline(false), + HashMap::new(), + HashMap::new(), + None, + true, + ); + let error = parser.next().unwrap().unwrap_err(); + assert_eq!(error.line(), 1); + assert_eq!(error.column(), 13); + assert_eq!(error.kind(), &ParseErrorKind::MultilineDisabled); + } + + #[test] + fn reports_the_whole_assignment_with_character_column() { + let error = parse("ключ value\n").unwrap_err(); + assert_eq!(error.line(), 1); + assert_eq!(error.column(), 6); + assert_eq!(error.byte_offset(), 9); + assert_eq!(error.kind(), &ParseErrorKind::MissingEquals); + assert_eq!(error.source_line(), "ключ value"); + } + + #[test] + fn strips_a_utf8_bom_from_every_entry_point() { + let parsed = parse("\u{feff}KEY=value\n").unwrap(); + assert_eq!(parsed, [("KEY".to_owned(), "value".to_owned())]); + } + + #[test] + fn rejects_nul_in_keys_and_values_before_environment_mutation() { + let key_error = parse("BAD\0KEY=value").unwrap_err(); + assert_eq!(key_error.kind(), &ParseErrorKind::InvalidKeyCharacter('\0')); + + let value_error = parse("KEY=bad\0value").unwrap_err(); + assert_eq!( + value_error.kind(), + &ParseErrorKind::InvalidValueCharacter('\0') + ); + } + + #[test] + fn preserves_escaped_trailing_whitespace() { + let parsed = parse("A=one\\ \nB=two\\\\ \nC=three\\ \t\nD=four\\ # comment\n").unwrap(); + assert_eq!(parsed[0].1, "one "); + assert_eq!(parsed[1].1, "two\\"); + assert_eq!(parsed[2].1, "three "); + assert_eq!(parsed[3].1, "four "); + } + + #[test] + fn reports_each_structural_error_kind() { + assert_eq!( + parse("=value").unwrap_err().kind(), + &ParseErrorKind::MissingKey + ); + assert_eq!( + parse("KEY value").unwrap_err().kind(), + &ParseErrorKind::MissingEquals + ); + assert_eq!( + parse("KEY=\"value\"tail").unwrap_err().kind(), + &ParseErrorKind::TrailingCharacters + ); + assert_eq!( + parse("KEY='value").unwrap_err().kind(), + &ParseErrorKind::UnterminatedQuote('\'') + ); + } + + #[test] + fn parser_stops_after_the_first_error() { + let mut parser = Parser::new( + "BROKEN value\nVALID=yes\n".to_owned(), + ParseOptions::new(), + HashMap::new(), + HashMap::new(), + None, + true, + ); + + assert!(parser.next().unwrap().is_err()); + assert!(parser.next().is_none()); + } + + #[test] + fn error_kind_messages_are_actionable() { + let cases = [ + ( + ParseErrorKind::MissingKey, + "expected an environment variable name", + ), + ( + ParseErrorKind::MissingEquals, + "expected `=` after the environment variable name", + ), + ( + ParseErrorKind::InvalidKeyCharacter('\0'), + "environment variable name contains invalid character '\\0'", + ), + ( + ParseErrorKind::InvalidValueCharacter('\0'), + "environment variable value contains invalid character '\\0'", + ), + ( + ParseErrorKind::TrailingCharacters, + "expected a comment or end of line after the quoted value", + ), + ( + ParseErrorKind::UnterminatedQuote('"'), + "unterminated '\"' quoted value", + ), + ( + ParseErrorKind::UnterminatedSubstitution, + "unterminated `${...}` substitution", + ), + ( + ParseErrorKind::InvalidSubstitution, + "invalid variable substitution", + ), + ( + ParseErrorKind::RequiredVariable { + name: "TOKEN".to_owned(), + message: String::new(), + }, + "required variable `TOKEN` is not set", + ), + ( + ParseErrorKind::RequiredVariable { + name: "TOKEN".to_owned(), + message: "provide one".to_owned(), + }, + "required variable `TOKEN` is not set: provide one", + ), + ( + ParseErrorKind::MultilineDisabled, + "multiline quoted values are disabled", + ), + ]; + + for (kind, expected) in cases { + assert_eq!(kind.to_string(), expected); + } + } + + #[test] + fn parses_empty_commented_values_and_export_as_a_regular_key() { + assert_eq!( + parse("EMPTY=# intentionally empty\n").unwrap(), + [("EMPTY".to_owned(), String::new())] + ); + + let error = parse("export").unwrap_err(); + assert_eq!(error.kind(), &ParseErrorKind::MissingEquals); + assert_eq!(error.column(), 7); + } + + #[test] + fn rejects_nul_inside_quoted_values() { + let error = parse("KEY=\"before\0after\"").unwrap_err(); + assert_eq!(error.kind(), &ParseErrorKind::InvalidValueCharacter('\0')); + assert_eq!(error.column(), 12); + } + + #[test] + fn unquoted_embedded_quotes_honor_escaped_quotes() { + let parsed = parse(r#"KEY=before "a\"b" after"#).unwrap(); + assert_eq!(parsed[0].1, r#"before "a"b" after"#); + } + + #[test] + fn preserves_literal_dollars_and_a_terminal_backslash() { + let parsed = parse_with_options( + "TRAILING_DOLLAR=$\nDIGIT=$9\nBACKSLASH=value\\", + ParseOptions::new().substitution(true), + ) + .unwrap(); + assert_eq!(parsed[0].1, "$"); + assert_eq!(parsed[1].1, "$9"); + assert_eq!(parsed[2].1, "value\\"); + } + + #[test] + fn escaped_closing_braces_do_not_end_nested_substitutions() { + let parsed = parse_with_options( + r"VALUE=${MISSING:-left\}right}", + ParseOptions::new().substitution(true), + ) + .unwrap(); + assert_eq!(parsed[0].1, r"left\}right"); + } + + #[test] + fn rejects_an_empty_braced_substitution() { + let error = + parse_with_options("VALUE=${}", ParseOptions::new().substitution(true)).unwrap_err(); + assert_eq!(error.kind(), &ParseErrorKind::InvalidSubstitution); + } + + #[test] + fn reports_errors_after_crlf_on_the_correct_line() { + let error = parse("FIRST=ok\r\nSECOND value").unwrap_err(); + assert_eq!(error.line(), 2); + assert_eq!(error.column(), 8); + assert_eq!(error.source_line(), "SECOND value"); + } + + #[test] + fn decodes_all_documented_double_quote_control_escapes() { + let parsed = parse(r#"VALUE="\a\b\f\r\v""#).unwrap(); + assert_eq!(parsed[0].1, "\u{0007}\u{0008}\u{000c}\r\u{000b}"); + } + + #[test] + fn reports_invalid_and_unterminated_substitutions() { + let options = ParseOptions::new().substitution(true); + assert_eq!( + parse_with_options("KEY=${VALUE", options) + .unwrap_err() + .kind(), + &ParseErrorKind::UnterminatedSubstitution + ); + assert_eq!( + parse_with_options("KEY=${9VALUE}", options) + .unwrap_err() + .kind(), + &ParseErrorKind::InvalidSubstitution + ); + assert_eq!( + parse_with_options("KEY=${VALUE:=default}", options) + .unwrap_err() + .kind(), + &ParseErrorKind::InvalidSubstitution + ); + } + + #[test] + fn supports_nested_and_all_alternative_substitution_forms() { + let parsed = parse_with_options( + "SET=value\nEMPTY=\n\ + NESTED=${MISSING:-${ALSO_MISSING:-fallback}}\n\ + SET_PLUS=${SET+alternate}\n\ + SET_COLON_PLUS=${SET:+alternate}\n\ + EMPTY_PLUS=${EMPTY+alternate}\n\ + EMPTY_COLON_PLUS=${EMPTY:+alternate}\n\ + EMPTY_QUESTION=${EMPTY?message}\n", + ParseOptions::new().substitution(true), + ) + .unwrap(); + let values: HashMap<_, _> = parsed.into_iter().collect(); + assert_eq!(values["NESTED"], "fallback"); + assert_eq!(values["SET_PLUS"], "alternate"); + assert_eq!(values["SET_COLON_PLUS"], "alternate"); + assert_eq!(values["EMPTY_PLUS"], "alternate"); + assert_eq!(values["EMPTY_COLON_PLUS"], ""); + assert_eq!(values["EMPTY_QUESTION"], ""); + } + + #[test] + fn required_substitution_distinguishes_unset_from_empty() { + let options = ParseOptions::new().substitution(true); + let unset = parse_with_options("KEY=${MISSING?set it}", options).unwrap_err(); + assert!(matches!( + unset.kind(), + ParseErrorKind::RequiredVariable { name, message } + if name == "MISSING" && message == "set it" + )); + + let empty = + parse_with_options("EMPTY=\nKEY=${EMPTY:?must not be empty}", options).unwrap_err(); + assert!(matches!( + empty.kind(), + ParseErrorKind::RequiredVariable { name, message } + if name == "EMPTY" && message == "must not be empty" + )); + } +} diff --git a/crates/mise-dotenv/src/render.rs b/crates/mise-dotenv/src/render.rs new file mode 100644 index 00000000000..cd22571c7d5 --- /dev/null +++ b/crates/mise-dotenv/src/render.rs @@ -0,0 +1,363 @@ +use crate::parse::is_horizontal_whitespace; +use std::{borrow::Cow, error, fmt}; + +/// An error encountered while rendering a dotenv assignment. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum RenderError { + /// Environment variable names cannot be empty. + EmptyKey, + /// An environment variable name contains a character unsupported by dotenv syntax. + InvalidKeyCharacter(char), + /// An environment variable value contains a character unsupported by dotenv syntax. + InvalidValueCharacter(char), +} + +impl fmt::Display for RenderError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EmptyKey => f.write_str("environment variable name cannot be empty"), + Self::InvalidKeyCharacter(character) => { + write!( + f, + "environment variable name contains invalid character {character:?}" + ) + } + Self::InvalidValueCharacter(character) => { + write!( + f, + "environment variable value contains invalid character {character:?}" + ) + } + } + } +} + +impl error::Error for RenderError {} + +/// Renders a value using the minimum quoting required by the default dotenv syntax. +/// +/// Values that already round-trip as unquoted input are returned unchanged and without an +/// allocation. Other values are double-quoted, with control characters and syntax-significant +/// characters escaped. Dollar signs remain unquoted when possible because variable substitution +/// is disabled by default. +/// +/// # Examples +/// +/// ``` +/// use dotenv_ng_core::render_value; +/// +/// assert_eq!(render_value("hello world")?.as_ref(), "hello world"); +/// assert_eq!(render_value(" trailing ")?.as_ref(), r#"" trailing ""#); +/// # Ok::<(), dotenv_ng_core::RenderError>(()) +/// ``` +pub fn render_value(value: &str) -> Result, RenderError> { + if value.contains('\0') { + return Err(RenderError::InvalidValueCharacter('\0')); + } + + if is_unquoted_safe(value) { + return Ok(Cow::Borrowed(value)); + } + + let mut rendered = String::with_capacity(value.len() + 2); + rendered.push('"'); + for character in value.chars() { + match character { + '\u{0007}' => rendered.push_str("\\a"), + '\u{0008}' => rendered.push_str("\\b"), + '\u{000c}' => rendered.push_str("\\f"), + '\n' => rendered.push_str("\\n"), + '\r' => rendered.push_str("\\r"), + '\t' => rendered.push_str("\\t"), + '\u{000b}' => rendered.push_str("\\v"), + '\\' => rendered.push_str("\\\\"), + '"' => rendered.push_str("\\\""), + '$' => rendered.push_str("\\$"), + _ => rendered.push(character), + } + } + rendered.push('"'); + Ok(Cow::Owned(rendered)) +} + +/// Renders one `key=value` assignment without a trailing newline. +/// +/// The key is validated against this crate's dotenv syntax. In particular, it must be non-empty +/// and cannot contain whitespace, `=`, `#`, or control characters. A key cannot begin with a +/// byte-order mark because dotenv parsers treat one at the start of a document as an encoding +/// marker rather than part of the key. +pub fn render_var(key: &str, value: &str) -> Result { + validate_key(key)?; + let value = render_value(value)?; + let mut rendered = String::with_capacity(key.len() + value.len() + 1); + rendered.push_str(key); + rendered.push('='); + rendered.push_str(&value); + Ok(rendered) +} + +/// Renders assignments as a dotenv document in iterator order. +/// +/// Assignments are separated by newlines. The returned document has no trailing newline. +/// Callers that need stable output from an unordered map should sort its entries first. +pub fn render(variables: I) -> Result +where + I: IntoIterator, + K: AsRef, + V: AsRef, +{ + let mut document = String::new(); + for (key, value) in variables { + if !document.is_empty() { + document.push('\n'); + } + document.push_str(&render_var(key.as_ref(), value.as_ref())?); + } + Ok(document) +} + +fn validate_key(key: &str) -> Result<(), RenderError> { + if key.is_empty() { + return Err(RenderError::EmptyKey); + } + if key.starts_with('\u{feff}') { + return Err(RenderError::InvalidKeyCharacter('\u{feff}')); + } + + if let Some(character) = key.chars().find(|character| { + matches!(character, '=' | '#') || character.is_whitespace() || character.is_control() + }) { + return Err(RenderError::InvalidKeyCharacter(character)); + } + + Ok(()) +} + +fn is_unquoted_safe(value: &str) -> bool { + let Some(first) = value.chars().next() else { + return true; + }; + if matches!(first, '\'' | '"' | '#' | '\n' | '\r') || is_horizontal_whitespace(first) { + return false; + } + if value + .chars() + .next_back() + .is_some_and(is_horizontal_whitespace) + { + return false; + } + + let mut quote = None; + let mut escaped = false; + let mut previous = None; + for character in value.chars() { + if matches!(character, '\0' | '\n' | '\r') { + return false; + } + + if let Some(open_quote) = quote { + if character == open_quote && !escaped { + quote = None; + } + if character == '\\' { + escaped = !escaped; + } else { + escaped = false; + } + } else { + if character == '#' && previous.is_none_or(is_horizontal_whitespace) && !escaped { + return false; + } + if matches!(character, '\'' | '"') && !escaped { + quote = Some(character); + } + escaped = character == '\\' && !escaped; + } + + previous = Some(character); + } + + let mut characters = value.chars(); + while let Some(character) = characters.next() { + if character != '\\' { + continue; + } + let Some(escaped) = characters.next() else { + break; + }; + if matches!(escaped, '\\' | '\'' | '"' | '$' | '#' | ' ' | '\t') { + return false; + } + } + + true +} + +#[cfg(test)] +mod tests { + use super::{RenderError, render, render_value, render_var}; + use crate::{EnvLoader, EnvMap, EnvSequence}; + use proptest::prelude::*; + use std::io::Cursor; + + fn parse(document: &str) -> Result { + EnvLoader::with_reader(Cursor::new(document)) + .sequence(EnvSequence::InputOnly) + .multiline(false) + .load() + } + + fn arbitrary_value() -> impl Strategy { + let unicode = any::().prop_filter("dotenv values cannot contain NUL", |value| { + !value.contains('\0') + }); + let syntax_heavy = proptest::string::string_regex(r"[\x01-\x7F]{0,64}") + .expect("the static property-test regular expression should be valid"); + prop_oneof![unicode, syntax_heavy] + } + + #[test] + fn leaves_values_unquoted_when_they_already_round_trip() { + for value in [ + "", + "plain", + "hello world", + r"C:\Users\me", + "value#literal", + "$2b$12$literal-bcrypt-hash", + "trailing\\", + r"unknown\qescape", + r"literal\nsequence", + "internal\twhitespace", + ] { + assert_eq!(render_value(value).unwrap().as_ref(), value); + } + } + + #[test] + fn quotes_and_escapes_values_only_when_required() { + for (value, expected) in [ + (" leading", r#"" leading""#), + ("trailing ", r#""trailing ""#), + ("#comment", "\"#comment\""), + ("value # comment", r#""value # comment""#), + (r#""quoted""#, r#""\"quoted\"""#), + ("line\nbreak", r#""line\nbreak""#), + ("carriage\rreturn", r#""carriage\rreturn""#), + (r"literal\ value", r#""literal\\ value""#), + (" leading $HOME", r#"" leading \$HOME""#), + ] { + assert_eq!(render_value(value).unwrap().as_ref(), expected); + } + } + + #[test] + fn rejects_unrepresentable_values_and_invalid_keys() { + assert_eq!( + render_value("nul\0value"), + Err(RenderError::InvalidValueCharacter('\0')) + ); + assert_eq!(render_var("", "value"), Err(RenderError::EmptyKey)); + + for (key, character) in [ + ("HAS SPACE", ' '), + ("HAS=EQUALS", '='), + ("HAS#HASH", '#'), + ("HAS\nNEWLINE", '\n'), + ("HAS\0NUL", '\0'), + ("\u{feff}BOM", '\u{feff}'), + ] { + assert_eq!( + render_var(key, "value"), + Err(RenderError::InvalidKeyCharacter(character)) + ); + } + } + + #[test] + fn render_error_messages_identify_the_invalid_field() { + assert_eq!( + RenderError::EmptyKey.to_string(), + "environment variable name cannot be empty" + ); + assert_eq!( + RenderError::InvalidKeyCharacter('\n').to_string(), + "environment variable name contains invalid character '\\n'" + ); + assert_eq!( + RenderError::InvalidValueCharacter('\0').to_string(), + "environment variable value contains invalid character '\\0'" + ); + } + + #[test] + fn escapes_all_supported_control_characters_when_quoting() { + let value = " \u{0007}\u{0008}\u{000c}\n\r\t\u{000b}"; + assert_eq!(render_value(value).unwrap(), r#"" \a\b\f\n\r\t\v""#); + assert_eq!( + parse(&format!("VALUE={}", render_value(value).unwrap())).unwrap()["VALUE"], + value + ); + } + + #[test] + fn renders_assignments_and_documents() { + assert_eq!( + render_var("naïve-key.name", "hello world").unwrap(), + "naïve-key.name=hello world" + ); + assert_eq!( + render_var("embedded\u{feff}bom", "value").unwrap(), + "embedded\u{feff}bom=value" + ); + assert_eq!( + render([("FIRST", "one"), ("SECOND", " two")]).unwrap(), + "FIRST=one\nSECOND=\" two\"" + ); + assert_eq!(render(Vec::<(&str, &str)>::new()).unwrap(), ""); + } + + proptest! { + #[test] + fn rendered_values_round_trip(value in arbitrary_value()) { + let assignment = render_var("PROPERTY", &value).unwrap(); + let parsed = parse(&assignment).unwrap(); + prop_assert_eq!(parsed.len(), 1); + prop_assert_eq!(parsed.get("PROPERTY").map(String::as_str), Some(value.as_str())); + } + + #[test] + fn values_are_quoted_exactly_when_raw_input_would_not_round_trip( + value in arbitrary_value() + ) { + let rendered = render_value(&value).unwrap(); + let raw_assignment = format!("PROPERTY={value}"); + let raw_round_trips = parse(&raw_assignment).is_ok_and(|parsed| { + parsed.len() == 1 + && parsed.get("PROPERTY").is_some_and(|parsed| parsed == &value) + }); + + prop_assert_eq!(rendered.as_ref() == value, raw_round_trips); + } + + #[test] + fn rendered_documents_round_trip(values in prop::collection::vec(arbitrary_value(), 0..32)) { + let document = render( + values + .iter() + .enumerate() + .map(|(index, value)| (format!("KEY_{index}"), value)), + ) + .unwrap(); + let parsed = parse(&document).unwrap(); + + prop_assert_eq!(parsed.len(), values.len()); + for (index, value) in values.iter().enumerate() { + let key = format!("KEY_{index}"); + prop_assert_eq!(parsed.get(&key), Some(value)); + } + } + } +} diff --git a/docs/environments/index.md b/docs/environments/index.md index c88da94adef..a12d9c6cd8f 100644 --- a/docs/environments/index.md +++ b/docs/environments/index.md @@ -429,7 +429,7 @@ _.file = { path = ".env.json", expand = true } ``` The `env_shell_expand` setting remains the global switch and can disable expansion even when a file -sets `expand = true`. Dotenv files retain dotenvy's normal same-file expansion behavior regardless; +sets `expand = true`. Dotenv files always expand references to earlier assignments in the same file, and a file's own values take precedence over variables that are already set (for example ones exported by `mise activate` from another `.env`); for dotenv files, `expand = true` additionally enables references to previously loaded values. ```toml diff --git a/e2e/env/test_env_file_ambient b/e2e/env/test_env_file_ambient new file mode 100644 index 00000000000..d55ef048d42 --- /dev/null +++ b/e2e/env/test_env_file_ambient @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2016 + +# A dotenv file's own values win over variables already in the environment when resolving `${VAR}`, +# e.g. ones exported by `mise activate` from another `.env`. +cat >mise.toml <<'EOF2' +[env] +_.file = [ + { path = ".env", expand = true }, + { path = ".env.test", expand = true }, +] +EOF2 +printf 'VAR_A="base"\nVAR_B="base_${VAR_A}"\nVAR_C="base_${VAR_B}"\n' >.env +printf 'VAR_A="test"\nVAR_B="test_${VAR_A}"\nVAR_C="test_${VAR_B}"\n' >.env.test + +for expand in true false; do + sed -i "s/expand = [a-z]*/expand = $expand/" mise.toml + export VAR_A=base VAR_B=base_base VAR_C=base_base_base + assert "mise env | grep '^export VAR_B='" "export VAR_B=test_test" + assert "mise env | grep '^export VAR_C='" "export VAR_C=test_test_test" + unset VAR_A VAR_B VAR_C + assert "mise env | grep '^export VAR_C='" "export VAR_C=test_test_test" +done diff --git a/src/config/env_directive/file.rs b/src/config/env_directive/file.rs index 896df58f5eb..023c846818d 100644 --- a/src/config/env_directive/file.rs +++ b/src/config/env_directive/file.rs @@ -242,71 +242,23 @@ impl EnvResults { return Ok(EnvMap::new()); } }; - if !expand { - // Preserve dotenvy's normal behavior unless cross-file expansion was - // explicitly requested. - let mut env = EnvMap::new(); - for item in dotenvy::from_read_iter(content.as_bytes()) { - let (k, v) = item.wrap_err_with(errfn)?; - env.insert(k, v); - } - return Ok(env); - } - // dotenvy substitutes `${VAR}` only against the process env + vars defined - // earlier in the same file and has no API for a custom map. Seed the parse - // with accumulated values, then retain only keys defined by this file. - let mut own_keys: std::collections::HashSet = std::collections::HashSet::new(); - for item in dotenvy::from_read_iter(content.as_bytes()) { - let (k, _v) = item.wrap_err_with(errfn)?; - own_keys.insert(k); - } - if own_keys.is_empty() { - return Ok(EnvMap::new()); - } - let mut prefix = String::new(); - for (k, v) in acc { - if own_keys.contains(k) || !is_env_key(k) { - continue; - } - prefix.push_str(k); - prefix.push_str("=\""); - prefix.push_str(&escape_dotenv_double_quoted(v)); - prefix.push_str("\"\n"); - } - let augmented = format!("{prefix}{content}"); + // `${VAR}` resolves against earlier assignments in this file first, so a file's own + // values are never shadowed by variables that happen to be set already (e.g. exported + // by `mise activate` from another `.env`). Only then do the surrounding values apply: + // everything loaded so far when `expand = true`, otherwise the process environment. + let outer: Vec<(String, String)> = if expand { + acc.iter().map(|(k, v)| (k.clone(), v.clone())).collect() + } else { + crate::env::vars_safe().collect() + }; let mut env = EnvMap::new(); - for item in dotenvy::from_read_iter(augmented.as_bytes()) { - let (k, v) = item.wrap_err_with(errfn)?; - if own_keys.contains(&k) { - env.insert(k, v); - } + for (k, v) in mise_dotenv::parse(&content, true, outer).wrap_err_with(errfn)? { + env.insert(k, v); } Ok(env) } } -fn is_env_key(k: &str) -> bool { - let mut chars = k.chars(); - chars - .next() - .is_some_and(|c| c.is_ascii_alphabetic() || c == '_') - && chars.all(|c| c.is_ascii_alphanumeric() || c == '_') -} - -fn escape_dotenv_double_quoted(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for c in s.chars() { - match c { - '\\' => out.push_str("\\\\"), - '"' => out.push_str("\\\""), - '$' => out.push_str("\\$"), - '\n' => out.push_str("\\n"), - _ => out.push(c), - } - } - out -} - #[cfg(test)] mod tests { use super::*; @@ -529,21 +481,6 @@ mod tests { restore_env_var("MISE_SOPS_ROPS", prev_rops); } - #[test] - fn escapes_seeded_dotenv_values() { - assert_eq!(escape_dotenv_double_quoted(r#"a$b"c\d"#), r#"a\$b\"c\\d"#); - assert_eq!(escape_dotenv_double_quoted("l1\nl2"), "l1\\nl2"); - } - - #[test] - fn validates_seeded_dotenv_keys() { - assert!(is_env_key("PGHOST")); - assert!(is_env_key("_FOO123")); - assert!(!is_env_key("1FOO")); - assert!(!is_env_key("FOO-BAR")); - assert!(!is_env_key("")); - } - #[tokio::test] async fn dotenv_reads_a_file_whatever_it_is_encoded_in() { // Windows PowerShell 5.1's `>` and `Out-File` write UTF-16LE by default, so the shell diff --git a/src/config/mod.rs b/src/config/mod.rs index db0e9d2c89e..16f0436d3ac 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1623,16 +1623,17 @@ impl Config { continue; } debug!("env_file: {}", display_path(&env_file)); - match dotenvy::from_path_iter(&env_file) { - Ok(iter) => { + match std::fs::read(&env_file) { + Ok(bytes) => { env_results.env_files.push(env_file.clone()); - for item in iter { - match item { - Ok((k, v)) => { + let content = String::from_utf8_lossy(&bytes); + match mise_dotenv::parse(&content, true, crate::env::vars_safe()) { + Ok(items) => { + for (k, v) in items { env_results.env.insert(k, (v, env_file.clone())); } - Err(err) => warn!("env_file: {err}"), } + Err(err) => warn!("env_file: {err}"), } } Err(err) => trace!("env_file: {err}"), From 02f52bcf71da56c49e583d7113b08fbebd2f22cf Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 16:07:57 +0000 Subject: [PATCH 2/7] fix(env): mark vendored dotenv Windows extern block unsafe Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_012ARBMJii5DSHj4MqUWLi8q --- crates/mise-dotenv/src/key.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/mise-dotenv/src/key.rs b/crates/mise-dotenv/src/key.rs index 12ba938c2e4..3ae1cb8479a 100644 --- a/crates/mise-dotenv/src/key.rs +++ b/crates/mise-dotenv/src/key.rs @@ -63,7 +63,7 @@ fn equivalent(left: &str, right: &str) -> bool { const TRUE: i32 = 1; #[link(name = "kernel32")] - extern "system" { + unsafe extern "system" { fn CompareStringOrdinal( left: *const u16, left_len: i32, From e576992a71782799f178898da62d62720cc0cccb Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 16:09:32 +0000 Subject: [PATCH 3/7] fix(env): box dotenv parse errors to satisfy clippy on Windows Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_012ARBMJii5DSHj4MqUWLi8q --- crates/mise-dotenv/src/err.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/mise-dotenv/src/err.rs b/crates/mise-dotenv/src/err.rs index 4aa5d936062..c998c719398 100644 --- a/crates/mise-dotenv/src/err.rs +++ b/crates/mise-dotenv/src/err.rs @@ -7,7 +7,7 @@ use crate::iter::ParseBufError; #[non_exhaustive] pub enum Error { /// The input did not conform to the supported dotenv syntax. - Parse(ParseError, Option), + Parse(Box, Option), /// An IO error may be encountered when reading from a file or reader. Io(io::Error, Option), /// The variable was not found in the environment. The `String` is the name of the variable. @@ -91,7 +91,7 @@ impl From for Error { impl From<(ParseBufError, Option)> for Error { fn from((e, path): (ParseBufError, Option)) -> Self { match e { - ParseBufError::Parse(error) => Self::Parse(error, path), + ParseBufError::Parse(error) => Self::Parse(Box::new(error), path), ParseBufError::Io(e) => Self::Io(e, path), } } @@ -108,7 +108,7 @@ mod tests { path::PathBuf, }; - fn parse_error() -> crate::ParseError { + fn parse_error() -> Box { let error = EnvLoader::with_reader(Cursor::new("BROKEN value")) .sequence(EnvSequence::InputOnly) .load() From 1c54a15e0d8128accee4bf43b9c4094b9de30e4e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 16:15:18 +0000 Subject: [PATCH 4/7] fix(env): reject undecodable env_file input instead of lossy decoding Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_012ARBMJii5DSHj4MqUWLi8q --- src/config/mod.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/config/mod.rs b/src/config/mod.rs index 16f0436d3ac..44b724c9193 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1626,14 +1626,20 @@ impl Config { match std::fs::read(&env_file) { Ok(bytes) => { env_results.env_files.push(env_file.clone()); - let content = String::from_utf8_lossy(&bytes); - match mise_dotenv::parse(&content, true, crate::env::vars_safe()) { - Ok(items) => { - for (k, v) in items { - env_results.env.insert(k, (v, env_file.clone())); + match file::decode_text(&bytes) { + Ok(content) => { + match mise_dotenv::parse(&content, true, crate::env::vars_safe()) { + Ok(items) => { + for (k, v) in items { + env_results.env.insert(k, (v, env_file.clone())); + } + } + Err(err) => warn!("env_file: {err}"), } } - Err(err) => warn!("env_file: {err}"), + Err(err) => { + warn!("env_file: ignoring {}: {err:#}", display_path(&env_file)) + } } } Err(err) => trace!("env_file: {err}"), From 24942c963705c6f2f896cfc4c1ea6918597c9823 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 16:20:31 +0000 Subject: [PATCH 5/7] chore(release): publish mise-dotenv with the other subcrates Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_012ARBMJii5DSHj4MqUWLi8q --- xtasks/release-plz | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xtasks/release-plz b/xtasks/release-plz index c1d08dd80b6..a0ba85e243e 100755 --- a/xtasks/release-plz +++ b/xtasks/release-plz @@ -382,6 +382,7 @@ if [[ $cur_version != "$latest_version" ]]; then MISE_SIGSTORE_VERSION="" MISE_CACHE_CORE_VERSION="" MISE_BREW_RELOCATION_VERSION="" + MISE_DOTENV_VERSION="" MISE_AGENT_ENV_VERSION="" MISE_SETTINGS_VERSION="" MISE_UTIL_VERSION="" @@ -402,6 +403,9 @@ if [[ $cur_version != "$latest_version" ]]; then bump_and_publish_subcrate "mise-brew-relocation" "crates/mise-brew-relocation" "MISE_BREW_RELOCATION_VERSION" cargo add -p mise --target 'cfg(unix)' "mise-brew-relocation@$MISE_BREW_RELOCATION_VERSION" + bump_and_publish_subcrate "mise-dotenv" "crates/mise-dotenv" "MISE_DOTENV_VERSION" + cargo add -p mise "mise-dotenv@$MISE_DOTENV_VERSION" + # Copy settings.toml into the crate directory so the published package includes it. cp settings.toml crates/mise-settings/settings.toml bump_and_publish_subcrate "mise-settings" "crates/mise-settings" "MISE_SETTINGS_VERSION" From 7456240a8bf280c1cddfe3dfe010841990ac432c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 16:29:31 +0000 Subject: [PATCH 6/7] docs(env): update dotenv parser references and name the skipped env_file Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_012ARBMJii5DSHj4MqUWLi8q --- crates/mise-dotenv/src/lib.rs | 2 +- crates/mise-dotenv/src/render.rs | 4 ++-- docs/environments/index.md | 5 ++--- src/config/mod.rs | 4 +++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/mise-dotenv/src/lib.rs b/crates/mise-dotenv/src/lib.rs index 14bfcc10cdd..492030a0d5e 100644 --- a/crates/mise-dotenv/src/lib.rs +++ b/crates/mise-dotenv/src/lib.rs @@ -148,7 +148,7 @@ fn environment_variables() -> Result { /// /// ``` /// let key = "HOME"; -/// match dotenv_ng_core::var(key) { +/// match mise_dotenv::var(key) { /// Ok(val) => println!("{key}: {val:?}"), /// Err(e) => println!("couldn't interpret {key}: {e}"), /// } diff --git a/crates/mise-dotenv/src/render.rs b/crates/mise-dotenv/src/render.rs index cd22571c7d5..584ea3b63f9 100644 --- a/crates/mise-dotenv/src/render.rs +++ b/crates/mise-dotenv/src/render.rs @@ -45,11 +45,11 @@ impl error::Error for RenderError {} /// # Examples /// /// ``` -/// use dotenv_ng_core::render_value; +/// use mise_dotenv::render_value; /// /// assert_eq!(render_value("hello world")?.as_ref(), "hello world"); /// assert_eq!(render_value(" trailing ")?.as_ref(), r#"" trailing ""#); -/// # Ok::<(), dotenv_ng_core::RenderError>(()) +/// # Ok::<(), mise_dotenv::RenderError>(()) /// ``` pub fn render_value(value: &str) -> Result, RenderError> { if value.contains('\0') { diff --git a/docs/environments/index.md b/docs/environments/index.md index a12d9c6cd8f..8fe718e5d95 100644 --- a/docs/environments/index.md +++ b/docs/environments/index.md @@ -389,9 +389,8 @@ _.file = '.env' ``` ::: info -Only dotenv-format files use [dotenvy](https://crates.io/crates/dotenvy) under the hood. If you have -problems with dotenv parsing, report them there rather than to mise, since there is not much mise can -do about how that crate works. JSON, YAML, and TOML files use separate parsers. +Dotenv-format files are parsed by mise's own `mise-dotenv` crate, derived from +[dotenv-ng](https://crates.io/crates/dotenv-ng-core). JSON, YAML, and TOML files use separate parsers. ::: The `env._.file` directive supports: diff --git a/src/config/mod.rs b/src/config/mod.rs index 44b724c9193..186e9f44968 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1634,7 +1634,9 @@ impl Config { env_results.env.insert(k, (v, env_file.clone())); } } - Err(err) => warn!("env_file: {err}"), + Err(err) => { + warn!("env_file: ignoring {}: {err}", display_path(&env_file)) + } } } Err(err) => { From 3500956ef007cd409a971721a4c30596d21354a7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 17:04:49 +0000 Subject: [PATCH 7/7] fix(env): keep literal ${...} text and assignments before a syntax error Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_012ARBMJii5DSHj4MqUWLi8q --- crates/mise-dotenv/src/lib.rs | 52 ++++++++++++++++++++++++++++++++- crates/mise-dotenv/src/parse.rs | 36 +++++++++++++---------- e2e/env/test_env_file_ambient | 13 +++++++++ src/config/mod.rs | 21 +++++++------ 4 files changed, 96 insertions(+), 26 deletions(-) diff --git a/crates/mise-dotenv/src/lib.rs b/crates/mise-dotenv/src/lib.rs index 492030a0d5e..b26e43cdbeb 100644 --- a/crates/mise-dotenv/src/lib.rs +++ b/crates/mise-dotenv/src/lib.rs @@ -189,6 +189,40 @@ where .map_err(|error| Error::from((error, None::))) } +/// Like [`parse`], but keeps the assignments read before a syntax error. +/// +/// Returns every assignment that parsed cleanly, in file order, plus the error that stopped +/// parsing, if any. Nothing after the error is read. +pub fn parse_partial( + content: &str, + substitution: bool, + substitutions: I, +) -> (Vec<(String, String)>, Option) +where + I: IntoIterator, +{ + let mut protected = HashMap::new(); + for (key, value) in substitutions { + key::insert(&mut protected, key, value); + } + let iter = Iter::new( + content.as_bytes(), + ParseOptions::new().substitution(substitution), + HashMap::new(), + protected, + None, + true, + ); + let mut items = Vec::new(); + for item in iter { + match item { + Ok(item) => items.push(item), + Err(error) => return (items, Some(Error::from((error, None::)))), + } + } + (items, None) +} + /// The sequence in which to load environment variables. /// /// Values in the latter override values in the former. @@ -482,7 +516,9 @@ impl<'a> EnvLoader<'a> { #[cfg(test)] mod tests { - use crate::{EnvLoader, EnvSequence, Error, Input, ParseErrorKind, ParseOptions, parse}; + use crate::{ + EnvLoader, EnvSequence, Error, Input, ParseErrorKind, ParseOptions, parse, parse_partial, + }; use std::path::Path; use std::{ env, fs, @@ -1049,6 +1085,20 @@ Line 6 assert_eq!(parsed[1], ("VAR_B".to_owned(), "test_test".to_owned())); } + #[test] + fn parse_partial_keeps_assignments_before_the_error() { + let (items, error) = parse_partial("A=1\nB=2\nBROKEN line\nC=3\n", true, []); + assert_eq!( + items, + [ + ("A".to_owned(), "1".to_owned()), + ("B".to_owned(), "2".to_owned()) + ] + ); + assert!(matches!(error, Some(Error::Parse(_, _)))); + assert!(parse_partial("A=1\n", true, []).1.is_none()); + } + #[test] fn parse_ignores_the_process_environment() { temp_env::with_var("DOTENV_NG_PARSE_AMBIENT", Some("ambient"), || { diff --git a/crates/mise-dotenv/src/parse.rs b/crates/mise-dotenv/src/parse.rs index 439e14b245e..bad1baeb81d 100644 --- a/crates/mise-dotenv/src/parse.rs +++ b/crates/mise-dotenv/src/parse.rs @@ -516,9 +516,17 @@ impl Parser { ) })?; let expression = &raw[expression_start..expression_end]; - let expanded = - self.evaluate_substitution(expression, source_offset + expression_start, style)?; - output.push_str(&expanded); + // Braced text that is not a substitution (`${VAR:=x}`, `${}`, ...) stays literal + // rather than failing the whole file; the dotenv parser mise used before did the same. + match self.evaluate_substitution(expression, source_offset + expression_start, style) { + Ok(expanded) => output.push_str(&expanded), + Err(error) if error.kind == ParseErrorKind::InvalidSubstitution => { + output.push_str("${"); + output.push_str(expression); + output.push('}'); + } + Err(error) => return Err(error), + } *position = expression_end + '}'.len_utf8(); return Ok(()); } @@ -1268,10 +1276,10 @@ LITERAL=$$FOO_BAR } #[test] - fn rejects_an_empty_braced_substitution() { - let error = - parse_with_options("VALUE=${}", ParseOptions::new().substitution(true)).unwrap_err(); - assert_eq!(error.kind(), &ParseErrorKind::InvalidSubstitution); + fn keeps_an_empty_braced_substitution_literal() { + let parsed = + parse_with_options("VALUE=${}", ParseOptions::new().substitution(true)).unwrap(); + assert_eq!(parsed[0].1, "${}"); } #[test] @@ -1289,7 +1297,7 @@ LITERAL=$$FOO_BAR } #[test] - fn reports_invalid_and_unterminated_substitutions() { + fn keeps_invalid_substitutions_literal_and_reports_unterminated_ones() { let options = ParseOptions::new().substitution(true); assert_eq!( parse_with_options("KEY=${VALUE", options) @@ -1298,16 +1306,12 @@ LITERAL=$$FOO_BAR &ParseErrorKind::UnterminatedSubstitution ); assert_eq!( - parse_with_options("KEY=${9VALUE}", options) - .unwrap_err() - .kind(), - &ParseErrorKind::InvalidSubstitution + parse_with_options("KEY=${9VALUE}", options).unwrap()[0].1, + "${9VALUE}" ); assert_eq!( - parse_with_options("KEY=${VALUE:=default}", options) - .unwrap_err() - .kind(), - &ParseErrorKind::InvalidSubstitution + parse_with_options("KEY=${VALUE:=default}", options).unwrap()[0].1, + "${VALUE:=default}" ); } diff --git a/e2e/env/test_env_file_ambient b/e2e/env/test_env_file_ambient index d55ef048d42..cba4b586b5f 100644 --- a/e2e/env/test_env_file_ambient +++ b/e2e/env/test_env_file_ambient @@ -21,3 +21,16 @@ for expand in true false; do unset VAR_A VAR_B VAR_C assert "mise env | grep '^export VAR_C='" "export VAR_C=test_test_test" done + +# The `env_file` setting keeps assignments read before a syntax error and does not shadow a +# file's own values with exported ones. +rm -f mise.toml +export OUTER_A=exported +printf 'OUTER_A=own\nOUTER_B="b_${OUTER_A}"\nBROKEN line\nOUTER_C=never\n' >.env.partial +assert "MISE_ENV_FILE=.env.partial mise x -- sh -c 'echo \$OUTER_B'" "b_own" +assert "MISE_ENV_FILE=.env.partial mise x -- sh -c 'echo \${OUTER_C:-unset}'" "unset" +unset OUTER_A + +# Braced text that is not a substitution stays literal instead of failing the file. +printf 'LIT=\"${LIT_X:=default}\"\nAFTER=ok\n' >.env.literal +assert "MISE_ENV_FILE=.env.literal mise x -- sh -c 'echo \$AFTER'" "ok" diff --git a/src/config/mod.rs b/src/config/mod.rs index 186e9f44968..c520bf386e3 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1628,15 +1628,18 @@ impl Config { env_results.env_files.push(env_file.clone()); match file::decode_text(&bytes) { Ok(content) => { - match mise_dotenv::parse(&content, true, crate::env::vars_safe()) { - Ok(items) => { - for (k, v) in items { - env_results.env.insert(k, (v, env_file.clone())); - } - } - Err(err) => { - warn!("env_file: ignoring {}: {err}", display_path(&env_file)) - } + // Keep the assignments that precede a syntax error, like a + // line-by-line loader would, and name the file cut short. + let (items, err) = + mise_dotenv::parse_partial(&content, true, crate::env::vars_safe()); + for (k, v) in items { + env_results.env.insert(k, (v, env_file.clone())); + } + if let Some(err) = err { + warn!( + "env_file: stopped reading {}: {err}", + display_path(&env_file) + ); } } Err(err) => {