Allow path = "${FOO}/bar" dependencies

src/cargo/core/features.rs

@@ -408,6 +408,8 @@ features! {
 
     // Allow specifying different binary name apart from the crate name
     (unstable, different_binary_name, "", "reference/unstable.html#different-binary-name"),
+
+    (unstable, expand_env_vars, "", "reference/unstable.html#expand-env-vars"),
 }
 
 pub struct Feature {
@@ -656,6 +658,7 @@ unstable_cli_options!(
     unstable_options: bool = ("Allow the usage of unstable options"),
     weak_dep_features: bool = ("Allow `dep_name?/feature` feature syntax"),
     skip_rustdoc_fingerprint: bool = (HIDDEN),
+    expand_env_vars: bool = ("Expand environment variable references like ${FOO} in certain contexts, such as `path = \"...\"` in dependencies."),
 );
 
 const STABILIZED_COMPILE_PROGRESS: &str = "The progress bar is now always \
@@ -878,6 +881,7 @@ impl CliUnstable {
             "extra-link-arg" => stabilized_warn(k, "1.56", STABILIZED_EXTRA_LINK_ARG),
             "configurable-env" => stabilized_warn(k, "1.56", STABILIZED_CONFIGURABLE_ENV),
             "future-incompat-report" => self.future_incompat_report = parse_empty(k, v)?,
+            "expand-env-vars" => self.expand_env_vars = parse_empty(k, v)?,
             _ => bail!("unknown `-Z` flag specified: {}", k),
         }
 

src/cargo/util/env_vars.rs

@@ -0,0 +1,251 @@
+//! Expands environment variable references in strings.
+
+use crate::util::CargoResult;
+use std::borrow::Cow;
+
+/// Expands a string, replacing references to variables with values provided by
+/// the caller.
+///
+/// This function looks for references to variables, similar to environment
+/// variable references in command-line shells or Makefiles, and replaces the
+/// references with values.  The caller provides a `query` function which gives
+/// the values of the variables.
+///
+/// The syntax used for variable references is `${name}` or `${name?default}` if
+/// a default value is provided. The curly braces are always required;
+/// `$FOO` will not be interpreted as a variable reference (and will be copied
+/// to the output).
+///
+/// If a variable is referenced, then it must have a value (`query` must return
+/// `Some`) or the variable reference must provide a default value (using the
+/// `...?default` syntax). If `query` returns `None` and the variable reference
+/// does not provide a default value, then the expansion of the entire string
+/// will fail and the function will return `Err`.
+///
+/// Most strings processed by Cargo will not contain variable references.
+/// Hence, this function uses `Cow<str>` for its return type; it will return
+/// its input string as `Cow::Borrowed` if no variable references were found.
+pub fn expand_vars_with<'a, Q>(s: &'a str, query: Q) -> CargoResult<Cow<'a, str>>
+where
+    Q: Fn(&str) -> CargoResult<Option<String>>,
+{
+    let mut rest: &str;
+    let mut result: String;
+    if let Some(pos) = s.find('$') {
+        result = String::with_capacity(s.len() + 50);
+        result.push_str(&s[..pos]);
+        rest = &s[pos..];
+    } else {
+        // Most strings do not contain environment variable references.
+        // We optimize for the case where there is no reference, and
+        // return the same (borrowed) string.
+        return Ok(Cow::Borrowed(s));
+    };
+
+    while let Some(pos) = rest.find('$') {
+        result.push_str(&rest[..pos]);
+        rest = &rest[pos..];
+        let mut chars = rest.chars();
+        let c0 = chars.next();
+        debug_assert_eq!(c0, Some('$')); // because rest.find()
+        match chars.next() {
+            Some('{') => {
+                // the expected case, which is handled below.
+            }
+            Some(c) => {
+                // We found '$' that was not paired with '{'.
+                // This is not a variable reference.
+                // Output the $ and continue.
+                result.push('$');
+                result.push(c);
+                rest = chars.as_str();
+                continue;
+            }
+            None => {
+                // We found '$' at the end of the string.
+                result.push('$');
+                break;
+            }
+        }
+        let name_start = chars.as_str();
+        let name: &str;
+        let default_value: Option<&str>;
+        // Look for '}' or '?'
+        loop {
+            let pos = name_start.len() - chars.as_str().len();
+            match chars.next() {
+                None => {
+                    anyhow::bail!("environment variable reference is missing closing brace.")
+                }
+                Some('}') => {
+                    name = &name_start[..pos];
+                    default_value = None;
+                    break;
+                }
+                Some('?') => {
+                    name = &name_start[..pos];
+                    let default_value_start = chars.as_str();
+                    loop {
+                        let pos = chars.as_str();
+                        if let Some(c) = chars.next() {
+                            if c == '}' {
+                                default_value = Some(
+                                    &default_value_start[..default_value_start.len() - pos.len()],
+                                );
+                                break;
+                            }
+                        } else {
+                            anyhow::bail!(
+                                "environment variable reference is missing closing brace."
+                            );
+                        }
+                    }
+                    break;
+                }
+                Some(_) => {
+                    // consume this character (as part of var name)
+                }
+            }
+        }
+
+        if name.is_empty() {
+            anyhow::bail!("environment variable reference has invalid empty name");
+        }
+        // We now have the environment variable name, and have parsed the end of the
+        // name reference.
+        match (query(name)?, default_value) {
+            (Some(value), _) => result.push_str(&value),
+            (None, Some(value)) => result.push_str(value),
+            (None, None) => anyhow::bail!(format!(
+                "environment variable '{}' is not set and has no default value",
+                name
+            )),
+        }
+        rest = chars.as_str();
+    }
+    result.push_str(rest);
+    Ok(Cow::Owned(result))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn basic() {
+        let query = |name: &str| {
+            Ok(Some(
+                match name {
+                    "FOO" => "/foo",
+                    "BAR" => "/bar",
+                    "FOO(ZAP)" => "/foo/zap",
+                    "WINKING_FACE" => "\u{1F609}",
+                    "\u{1F916}" => "ROBOT FACE",
+                    _ => return Ok(None),
+                }
+                .to_string(),
+            ))
+        };
+
+        let expand = |s| expand_vars_with(s, query);
+
+        macro_rules! case {
+            ($input:expr, $expected_output:expr) => {{
+                let input = $input;
+                let expected_output = $expected_output;
+                match expand(input) {
+                    Ok(output) => {
+                        assert_eq!(output, expected_output, "input = {:?}", input);
+                    }
+                    Err(e) => {
+                        panic!(
+                            "Expected string {:?} to expand successfully, but it failed: {:?}",
+                            input, e
+                        );
+                    }
+                }
+            }};
+        }
+
+        macro_rules! err_case {
+            ($input:expr, $expected_error:expr) => {{
+                let input = $input;
+                let expected_error = $expected_error;
+                match expand(input) {
+                    Ok(output) => {
+                        panic!("Expected expansion of string {:?} to fail, but it succeeded with value {:?}", input, output);
+                    }
+                    Err(e) => {
+                        let message = e.to_string();
+                        assert_eq!(message, expected_error, "input = {:?}", input);
+                    }
+                }
+            }}
+        }
+
+        // things without references should not change.
+        case!("", "");
+        case!("identity", "identity");
+
+        // we require ${...} (braces), so we ignore $FOO.
+        case!("$FOO/some_package", "$FOO/some_package");
+
+        // make sure variable references at the beginning, middle, and end
+        // of a string all work correctly.
+        case!("${FOO}", "/foo");
+        case!("${FOO} one", "/foo one");
+        case!("one ${FOO}", "one /foo");
+        case!("one ${FOO} two", "one /foo two");
+        case!("one ${FOO} two ${BAR} three", "one /foo two /bar three");
+
+        // variable names can contain most characters, except for '}' or '?'
+        // (Windows sets "ProgramFiles(x86)", for example.)
+        case!("${FOO(ZAP)}", "/foo/zap");
+
+        // variable is set, and has a default (which goes unused)
+        case!("${FOO?/default}", "/foo");
+
+        // variable is not set, but does have default
+        case!("${VAR_NOT_SET?/default}", "/default");
+
+        // variable is not set and has no default
+        err_case!(
+            "${VAR_NOT_SET}",
+            "environment variable 'VAR_NOT_SET' is not set and has no default value"
+        );
+
+        // environment variables with unicode values are ok
+        case!("${WINKING_FACE}", "\u{1F609}");
+
+        // strings with unicode in them are ok
+        case!("\u{1F609}${FOO}", "\u{1F609}/foo");
+
+        // environment variable names with unicode in them are ok
+        case!("${\u{1F916}}", "ROBOT FACE");
+
+        // default values with unicode in them are ok
+        case!("${VAR_NOT_SET?\u{1F916}}", "\u{1F916}");
+
+        // invalid names
+        err_case!(
+            "${}",
+            "environment variable reference has invalid empty name"
+        );
+        err_case!(
+            "${?default}",
+            "environment variable reference has invalid empty name"
+        );
+        err_case!(
+            "${",
+            "environment variable reference is missing closing brace."
+        );
+        err_case!(
+            "${FOO",
+            "environment variable reference is missing closing brace."
+        );
+        err_case!(
+            "${FOO?default",
+            "environment variable reference is missing closing brace."
+        );
+    }
+}

src/cargo/util/mod.rs

@@ -35,6 +35,7 @@ mod counter;
 pub mod cpu;
 mod dependency_queue;
 pub mod diagnostic_server;
+pub mod env_vars;
 pub mod errors;
 mod flock;
 pub mod graph;

src/cargo/util/toml/mod.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::fmt;
 use std::marker::PhantomData;
@@ -251,19 +252,60 @@ impl<'de, P: Deserialize<'de>> de::Deserialize<'de> for TomlDependency<P> {
     }
 }
 
+/// Expands any environment variables found in `string`.
+///
+/// Expanding environment variables is currently an unstable feature.
+/// If the feature is not enabled, then we do not expand variable
+/// references, so "${FOO}" remains exactly "${FOO}" in the output.
+///
+/// This feature may be in one of _three_ modes:
+/// * Completely disabled. This is the default (stable) behavior. In this mode,
+///   variable references in `string` are ignored (treated like normal text).
+/// * `-Z expand_env_vars` was specified on the command-line, but the current
+///   manifest does not specify `cargo-features = ["expand_env_vars"]. In this
+///   mode, we do look for variable references. If we find _any_ variable
+///   reference, then we report an error. This mode is intended only to allow us
+///   to do Crater runs, so that we can find any projects that this new
+///   feature would break, since they contain strings that match our `${FOO}` syntax.
+/// * The current manifest contains `cargo-features = ["expand_env_vars"]`.
+///   In this mode, we look for variable references and process them.
+///   This is the mode that will eventually be stabilized and become the default.
+fn expand_env_var<'a>(
+    string: &'a str,
+    config: &Config,
+    features: &Features,
+) -> CargoResult<Cow<'a, str>> {
+    if config.cli_unstable().expand_env_vars || features.is_enabled(Feature::expand_env_vars()) {
+        let expanded_path = crate::util::env_vars::expand_vars_with(string, |name| {
+            if features.is_enabled(Feature::expand_env_vars()) {
+                Ok(std::env::var(name).ok())
+            } else {
+                // This manifest contains a string which looks like a variable
+                // reference, but the manifest has not enabled the feature.
+                // Report an error.
+                anyhow::bail!("this manifest uses environment variable references (e.g. ${FOO}) but has not specified `cargo-features = [\"expand-env-vars\"]`.");
+            }
+        })?;
+        Ok(expanded_path)
+    } else {
+        Ok(Cow::Borrowed(string))
+    }
+}
+
 pub trait ResolveToPath {
-    fn resolve(&self, config: &Config) -> PathBuf;
+    fn resolve(&self, config: &Config, features: &Features) -> CargoResult<PathBuf>;
 }
 
 impl ResolveToPath for String {
-    fn resolve(&self, _: &Config) -> PathBuf {
-        self.into()
+    fn resolve(&self, config: &Config, features: &Features) -> CargoResult<PathBuf> {
+        let expanded = expand_env_var(self, config, features)?;
+        Ok(expanded.to_string().into())
     }
 }
 
 impl ResolveToPath for ConfigRelativePath {
-    fn resolve(&self, c: &Config) -> PathBuf {
-        self.resolve_path(c)
+    fn resolve(&self, c: &Config, _features: &Features) -> CargoResult<PathBuf> {
+        Ok(self.resolve_path(c))
     }
 }
 
@@ -1887,7 +1929,7 @@ impl<P: ResolveToPath> DetailedTomlDependency<P> {
                 SourceId::for_git(&loc, reference)?
             }
             (None, Some(path), _, _) => {
-                let path = path.resolve(cx.config);
+                let path = path.resolve(cx.config, cx.features)?;
                 cx.nested_paths.push(path.clone());
                 // If the source ID for the package we're parsing is a path
                 // source, then we normalize the path here to get rid of

src/doc/src/reference/unstable.md

@@ -1328,6 +1328,29 @@ filename = "007bar"
 path = "src/main.rs"
 ```
 
+### expand-env-vars
+
+The `expand-env-vars` feature allows certain fields within manifests to refer
+to environment variables, using `${FOO}` syntax. Currently, the only field that
+may refer to environment variables is the `path` field of a dependency.
+
+For example:
+
+```toml
+cargo-features = ["expand-env-vars"]
+
+[dependencies]
+foo = { path = "${MY_PROJECT_ROOT}/some/long/path/utils" }
+```
+
+For organizations that manage large code bases, using relative walk-up paths
+(e.g. `../../../../long/path/utils`) is not practical; some organizations
+forbid using walk-up paths. The `expand-env-vars` feature allows Cargo
+to work in such systems.
+
+The curly braces are required; `$FOO` will not be interpreted as a variable
+reference.
+
 ## Stabilized and removed features
 
 ### Compile progress