Support environment variables in index URLs in requirements files (#2036

)

## Summary

This also preserves the environment variables in the output file, e.g.:

```
Resolved 1 package in 216ms
# This file was autogenerated by uv via the following command:
#    uv pip compile requirements.in --emit-index-url
--index-url https://test.pypi.org/${SUFFIX}

requests==2.5.4.1
```

I'm torn on whether that's correct or undesirable here.

Closes #2035.

crates/distribution-types/src/index_url.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::fmt::{Display, Formatter};
 use std::ops::Deref;
 use std::path::PathBuf;
@@ -8,39 +9,59 @@ use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 use url::Url;
 
-use pep508_rs::{split_scheme, Scheme};
+use pep508_rs::{split_scheme, Scheme, VerbatimUrl};
 use uv_fs::normalize_url_path;
 
+use crate::Verbatim;
+
 static PYPI_URL: Lazy<Url> = Lazy::new(|| Url::parse("https://pypi.org/simple").unwrap());
 
+static DEFAULT_INDEX_URL: Lazy<IndexUrl> =
+    Lazy::new(|| IndexUrl::Pypi(VerbatimUrl::from_url(PYPI_URL.clone())));
+
 /// The url of an index, newtype'd to avoid mixing it with file urls.
 #[derive(Debug, Clone, Hash, Eq, PartialEq, Serialize, Deserialize)]
 pub enum IndexUrl {
-    Pypi,
-    Url(Url),
+    Pypi(VerbatimUrl),
+    Url(VerbatimUrl),
 }
 
 impl Display for IndexUrl {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
-            Self::Pypi => Display::fmt(&*PYPI_URL, f),
+            Self::Pypi(url) => Display::fmt(url, f),
             Self::Url(url) => Display::fmt(url, f),
         }
     }
 }
 
+impl Verbatim for IndexUrl {
+    fn verbatim(&self) -> Cow<'_, str> {
+        match self {
+            Self::Pypi(url) => url.verbatim(),
+            Self::Url(url) => url.verbatim(),
+        }
+    }
+}
+
 impl FromStr for IndexUrl {
     type Err = url::ParseError;
 
-    fn from_str(url: &str) -> Result<Self, Self::Err> {
-        Ok(Self::from(Url::parse(url)?))
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let url = Url::parse(s)?;
+        let url = VerbatimUrl::from_url(url).with_given(s.to_owned());
+        if *url.raw() == *PYPI_URL {
+            Ok(Self::Pypi(url))
+        } else {
+            Ok(Self::Url(url))
+        }
     }
 }
 
-impl From<Url> for IndexUrl {
-    fn from(url: Url) -> Self {
-        if url == *PYPI_URL {
-            Self::Pypi
+impl From<VerbatimUrl> for IndexUrl {
+    fn from(url: VerbatimUrl) -> Self {
+        if *url.raw() == *PYPI_URL {
+            Self::Pypi(url)
         } else {
             Self::Url(url)
         }
@@ -50,8 +71,8 @@ impl From<Url> for IndexUrl {
 impl From<IndexUrl> for Url {
     fn from(index: IndexUrl) -> Self {
         match index {
-            IndexUrl::Pypi => PYPI_URL.clone(),
-            IndexUrl::Url(url) => url,
+            IndexUrl::Pypi(url) => url.to_url(),
+            IndexUrl::Url(url) => url.to_url(),
         }
     }
 }
@@ -61,7 +82,7 @@ impl Deref for IndexUrl {
 
     fn deref(&self) -> &Self::Target {
         match &self {
-            Self::Pypi => &PYPI_URL,
+            Self::Pypi(url) => url,
             Self::Url(url) => url,
         }
     }
@@ -152,7 +173,7 @@ impl Default for IndexLocations {
     /// By default, use the `PyPI` index.
     fn default() -> Self {
         Self {
-            index: Some(IndexUrl::Pypi),
+            index: Some(DEFAULT_INDEX_URL.clone()),
             extra_index: Vec::new(),
             flat_index: Vec::new(),
             no_index: false,
@@ -211,7 +232,7 @@ impl<'a> IndexLocations {
         } else {
             match self.index.as_ref() {
                 Some(index) => Some(index),
-                None => Some(&IndexUrl::Pypi),
+                None => Some(&DEFAULT_INDEX_URL),
             }
         }
     }
@@ -259,7 +280,7 @@ impl Default for IndexUrls {
     /// By default, use the `PyPI` index.
     fn default() -> Self {
         Self {
-            index: Some(IndexUrl::Pypi),
+            index: Some(DEFAULT_INDEX_URL.clone()),
             extra_index: Vec::new(),
             no_index: false,
         }
@@ -278,7 +299,7 @@ impl<'a> IndexUrls {
         } else {
             match self.index.as_ref() {
                 Some(index) => Some(index),
-                None => Some(&IndexUrl::Pypi),
+                None => Some(&DEFAULT_INDEX_URL),
             }
         }
     }

crates/pep508-rs/src/lib.rs

@@ -755,11 +755,11 @@ fn preprocess_url(
                 #[cfg(feature = "non-pep508-extensions")]
                 if let Some(working_dir) = working_dir {
                     return Ok(
-                        VerbatimUrl::from_path(path, working_dir).with_given(url.to_string())
+                        VerbatimUrl::parse_path(path, working_dir).with_given(url.to_string())
                     );
                 }
 
-                Ok(VerbatimUrl::from_absolute_path(path)
+                Ok(VerbatimUrl::parse_absolute_path(path)
                     .map_err(|err| Pep508Error {
                         message: Pep508ErrorSource::UrlError(err),
                         start,
@@ -783,10 +783,12 @@ fn preprocess_url(
             _ => {
                 #[cfg(feature = "non-pep508-extensions")]
                 if let Some(working_dir) = working_dir {
-                    return Ok(VerbatimUrl::from_path(url, working_dir).with_given(url.to_string()));
+                    return Ok(
+                        VerbatimUrl::parse_path(url, working_dir).with_given(url.to_string())
+                    );
                 }
 
-                Ok(VerbatimUrl::from_absolute_path(url)
+                Ok(VerbatimUrl::parse_absolute_path(url)
                     .map_err(|err| Pep508Error {
                         message: Pep508ErrorSource::UrlError(err),
                         start,
@@ -800,10 +802,10 @@ fn preprocess_url(
         // Ex) `../editable/`
         #[cfg(feature = "non-pep508-extensions")]
         if let Some(working_dir) = working_dir {
-            return Ok(VerbatimUrl::from_path(url, working_dir).with_given(url.to_string()));
+            return Ok(VerbatimUrl::parse_path(url, working_dir).with_given(url.to_string()));
         }
 
-        Ok(VerbatimUrl::from_absolute_path(url)
+        Ok(VerbatimUrl::parse_absolute_path(url)
             .map_err(|err| Pep508Error {
                 message: Pep508ErrorSource::UrlError(err),
                 start,

crates/pep508-rs/src/verbatim_url.rs

@@ -5,7 +5,7 @@ use std::path::{Component, Path, PathBuf};
 
 use once_cell::sync::Lazy;
 use regex::Regex;
-use url::Url;
+use url::{ParseError, Url};
 
 /// A wrapper around [`Url`] that preserves the original string.
 #[derive(Debug, Clone, Eq, derivative::Derivative)]
@@ -29,15 +29,26 @@ pub struct VerbatimUrl {
 
 impl VerbatimUrl {
     /// Parse a URL from a string, expanding any environment variables.
-    pub fn parse(given: impl AsRef<str>) -> Result<Self, VerbatimUrlError> {
-        let url = Url::parse(&expand_env_vars(given.as_ref(), true))
-            .map_err(|err| VerbatimUrlError::Url(given.as_ref().to_owned(), err))?;
+    pub fn parse(given: impl AsRef<str>) -> Result<Self, ParseError> {
+        let url = Url::parse(&expand_env_vars(given.as_ref(), true))?;
         Ok(Self { url, given: None })
     }
 
+    /// Create a [`VerbatimUrl`] from a [`Url`].
+    pub fn from_url(url: Url) -> Self {
+        Self { url, given: None }
+    }
+
+    /// Create a [`VerbatimUrl`] from a file path.
+    pub fn from_path(path: impl AsRef<Path>) -> Self {
+        let path = normalize_path(path.as_ref());
+        let url = Url::from_file_path(path).expect("path is absolute");
+        Self { url, given: None }
+    }
+
     /// Parse a URL from an absolute or relative path.
     #[cfg(feature = "non-pep508-extensions")] // PEP 508 arguably only allows absolute file URLs.
-    pub fn from_path(path: impl AsRef<str>, working_dir: impl AsRef<Path>) -> Self {
+    pub fn parse_path(path: impl AsRef<str>, working_dir: impl AsRef<Path>) -> Self {
         // Expand any environment variables.
         let path = PathBuf::from(expand_env_vars(path.as_ref(), false).as_ref());
 
@@ -58,7 +69,7 @@ impl VerbatimUrl {
     }
 
     /// Parse a URL from an absolute path.
-    pub fn from_absolute_path(path: impl AsRef<str>) -> Result<Self, VerbatimUrlError> {
+    pub fn parse_absolute_path(path: impl AsRef<str>) -> Result<Self, VerbatimUrlError> {
         // Expand any environment variables.
         let path = PathBuf::from(expand_env_vars(path.as_ref(), false).as_ref());
 
@@ -115,7 +126,9 @@ impl std::str::FromStr for VerbatimUrl {
     type Err = VerbatimUrlError;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        Self::parse(s).map(|url| url.with_given(s.to_owned()))
+        Self::parse(s)
+            .map(|url| url.with_given(s.to_owned()))
+            .map_err(|e| VerbatimUrlError::Url(s.to_owned(), e))
     }
 }
 
@@ -138,7 +151,7 @@ impl Deref for VerbatimUrl {
 pub enum VerbatimUrlError {
     /// Failed to parse a URL.
     #[error("{0}")]
-    Url(String, #[source] url::ParseError),
+    Url(String, #[source] ParseError),
 
     /// Received a relative path, but no working directory was provided.
     #[error("relative path without a working directory: {0}")]

crates/requirements-txt/src/lib.rs

@@ -70,9 +70,9 @@ enum RequirementsTxtStatement {
     /// `-e`
     EditableRequirement(EditableRequirement),
     /// `--index-url`
-    IndexUrl(Url),
+    IndexUrl(VerbatimUrl),
     /// `--extra-index-url`
-    ExtraIndexUrl(Url),
+    ExtraIndexUrl(VerbatimUrl),
     /// `--find-links`
     FindLinks(FindLink),
     /// `--no-index`
@@ -215,7 +215,7 @@ impl EditableRequirement {
                     // Transform, e.g., `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz`.
                     let path = normalize_url_path(path);
 
-                    VerbatimUrl::from_path(path, working_dir.as_ref())
+                    VerbatimUrl::parse_path(path, working_dir.as_ref())
                 }
 
                 // Ex) `https://download.pytorch.org/whl/torch_stable.html`
@@ -226,11 +226,11 @@ impl EditableRequirement {
                 }
 
                 // Ex) `C:/Users/ferris/wheel-0.42.0.tar.gz`
-                _ => VerbatimUrl::from_path(requirement, working_dir.as_ref()),
+                _ => VerbatimUrl::parse_path(requirement, working_dir.as_ref()),
             }
         } else {
             // Ex) `../editable/`
-            VerbatimUrl::from_path(requirement, working_dir.as_ref())
+            VerbatimUrl::parse_path(requirement, working_dir.as_ref())
         };
 
         // Create a `PathBuf`.
@@ -308,9 +308,9 @@ pub struct RequirementsTxt {
     /// Editables with `-e`.
     pub editables: Vec<EditableRequirement>,
     /// The index URL, specified with `--index-url`.
-    pub index_url: Option<Url>,
+    pub index_url: Option<VerbatimUrl>,
     /// The extra index URLs, specified with `--extra-index-url`.
-    pub extra_index_urls: Vec<Url>,
+    pub extra_index_urls: Vec<VerbatimUrl>,
     /// The find links locations, specified with `--find-links`.
     pub find_links: Vec<FindLink>,
     /// Whether to ignore the index, specified with `--no-index`.
@@ -482,22 +482,26 @@ fn parse_entry(
             .map_err(|err| err.with_offset(start))?;
         RequirementsTxtStatement::EditableRequirement(editable_requirement)
     } else if s.eat_if("-i") || s.eat_if("--index-url") {
-        let url = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?;
-        let url = Url::parse(url).map_err(|err| RequirementsTxtParserError::Url {
-            source: err,
-            url: url.to_string(),
-            start,
-            end: s.cursor(),
-        })?;
+        let given = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?;
+        let url = VerbatimUrl::parse(given)
+            .map(|url| url.with_given(given.to_owned()))
+            .map_err(|err| RequirementsTxtParserError::Url {
+                source: err,
+                url: given.to_string(),
+                start,
+                end: s.cursor(),
+            })?;
         RequirementsTxtStatement::IndexUrl(url)
     } else if s.eat_if("--extra-index-url") {
-        let url = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?;
-        let url = Url::parse(url).map_err(|err| RequirementsTxtParserError::Url {
-            source: err,
-            url: url.to_string(),
-            start,
-            end: s.cursor(),
-        })?;
+        let given = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?;
+        let url = VerbatimUrl::parse(given)
+            .map(|url| url.with_given(given.to_owned()))
+            .map_err(|err| RequirementsTxtParserError::Url {
+                source: err,
+                url: given.to_string(),
+                start,
+                end: s.cursor(),
+            })?;
         RequirementsTxtStatement::ExtraIndexUrl(url)
     } else if s.eat_if("--no-index") {
         RequirementsTxtStatement::NoIndex

crates/uv-cache/src/wheel.rs

@@ -30,7 +30,7 @@ pub enum WheelCache<'a> {
 impl<'a> WheelCache<'a> {
     fn bucket(&self) -> PathBuf {
         match self {
-            WheelCache::Index(IndexUrl::Pypi) => WheelCacheKind::Pypi.root(),
+            WheelCache::Index(IndexUrl::Pypi(_)) => WheelCacheKind::Pypi.root(),
             WheelCache::Index(url) => WheelCacheKind::Index
                 .root()
                 .join(digest(&CanonicalUrl::new(url))),

crates/uv-client/src/flat_index.rs

@@ -14,6 +14,7 @@ use distribution_types::{
     RegistryBuiltDist, RegistrySourceDist, SourceDist,
 };
 use pep440_rs::Version;
+use pep508_rs::VerbatimUrl;
 use platform_tags::Tags;
 use pypi_types::{Hashes, Yanked};
 use uv_auth::safe_copy_url_auth;
@@ -191,13 +192,14 @@ impl<'a> FlatIndexClient<'a> {
             .await;
         match response {
             Ok(files) => {
+                let index_url = IndexUrl::Url(VerbatimUrl::from_url(url.clone()));
                 let files = files
                     .into_iter()
                     .filter_map(|file| {
                         Some((
                             DistFilename::try_from_normalized_filename(&file.filename)?,
                             file,
-                            IndexUrl::Url(url.clone()),
+                            index_url.clone(),
                         ))
                     })
                     .collect();
@@ -214,6 +216,7 @@ impl<'a> FlatIndexClient<'a> {
     fn read_from_directory(path: &PathBuf) -> Result<FlatIndexEntries, std::io::Error> {
         // Absolute paths are required for the URL conversion.
         let path = fs_err::canonicalize(path)?;
+        let index_url = IndexUrl::Url(VerbatimUrl::from_path(&path));
 
         let mut dists = Vec::new();
         for entry in fs_err::read_dir(path)? {
@@ -249,7 +252,7 @@ impl<'a> FlatIndexClient<'a> {
                 );
                 continue;
             };
-            dists.push((filename, file, IndexUrl::Pypi));
+            dists.push((filename, file, index_url.clone()));
         }
         Ok(FlatIndexEntries::from_entries(dists))
     }

crates/uv-client/src/registry_client.rs

@@ -237,7 +237,7 @@ impl RegistryClient {
         let cache_entry = self.cache.entry(
             CacheBucket::Simple,
             Path::new(&match index {
-                IndexUrl::Pypi => "pypi".to_string(),
+                IndexUrl::Pypi(_) => "pypi".to_string(),
                 IndexUrl::Url(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)),
             }),
             format!("{package_name}.rkyv"),