Implement experimental registry HTTP API from RFC

crates/cargo-test-support/src/lib.rs

@@ -1525,6 +1525,7 @@ fn substitute_macros(input: &str) -> String {
         ("[DOCUMENTING]", " Documenting"),
         ("[FRESH]", "       Fresh"),
         ("[UPDATING]", "    Updating"),
+        ("[PREFETCHING]", " Prefetching"),
         ("[ADDING]", "      Adding"),
         ("[REMOVING]", "    Removing"),
         ("[DOCTEST]", "   Doc-tests"),

crates/cargo-test-support/src/registry.rs

@@ -7,7 +7,12 @@ use flate2::Compression;
 use std::collections::HashMap;
 use std::fs::{self, File};
 use std::io::prelude::*;
+use std::io::BufReader;
+use std::net::{SocketAddr, TcpListener};
 use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::thread;
 use tar::{Builder, Header};
 use url::Url;
 
@@ -213,6 +218,165 @@ pub fn init() {
     );
 }
 
+pub struct RegistryServer {
+    done: Arc<AtomicBool>,
+    server: Option<thread::JoinHandle<()>>,
+    addr: SocketAddr,
+}
+
+impl RegistryServer {
+    pub fn addr(&self) -> SocketAddr {
+        self.addr
+    }
+}
+
+impl Drop for RegistryServer {
+    fn drop(&mut self) {
+        self.done.store(true, Ordering::SeqCst);
+        // NOTE: we can't actually await the server since it's blocked in accept()
+        let _ = self.server.take().unwrap();
+    }
+}
+
+#[must_use]
+pub fn serve_registry(registry_path: PathBuf) -> RegistryServer {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let addr = listener.local_addr().unwrap();
+    let done = Arc::new(AtomicBool::new(false));
+    let done2 = done.clone();
+
+    let t = thread::spawn(move || {
+        let mut line = String::new();
+        'server: while !done2.load(Ordering::SeqCst) {
+            let (socket, _) = listener.accept().unwrap();
+            // Let's implement a very naive static file HTTP server.
+            let mut buf = BufReader::new(socket);
+
+            // First, the request line:
+            // GET /path HTTPVERSION
+            line.clear();
+            if buf.read_line(&mut line).unwrap() == 0 {
+                // Connection terminated.
+                continue;
+            }
+
+            assert!(line.starts_with("GET "), "got non-GET request: {}", line);
+            let path = PathBuf::from(
+                line.split_whitespace()
+                    .skip(1)
+                    .next()
+                    .unwrap()
+                    .trim_start_matches('/'),
+            );
+
+            let file = registry_path.join(path);
+            if file.exists() {
+                // Grab some other headers we may care about.
+                let mut if_modified_since = None;
+                let mut if_none_match = None;
+                loop {
+                    line.clear();
+                    if buf.read_line(&mut line).unwrap() == 0 {
+                        continue 'server;
+                    }
+
+                    if line == "\r\n" {
+                        // End of headers.
+                        line.clear();
+                        break;
+                    }
+
+                    let value = line
+                        .splitn(2, ':')
+                        .skip(1)
+                        .next()
+                        .map(|v| v.trim())
+                        .unwrap();
+
+                    if line.starts_with("If-Modified-Since:") {
+                        if_modified_since = Some(value.to_owned());
+                    } else if line.starts_with("If-None-Match:") {
+                        if_none_match = Some(value.trim_matches('"').to_owned());
+                    }
+                }
+
+                // Now grab info about the file.
+                let data = fs::read(&file).unwrap();
+                let etag = Sha256::new().update(&data).finish_hex();
+                let last_modified = format!("{:?}", file.metadata().unwrap().modified().unwrap());
+
+                // Start to construct our response:
+                let mut any_match = false;
+                let mut all_match = true;
+                if let Some(expected) = if_none_match {
+                    if etag != expected {
+                        all_match = false;
+                    } else {
+                        any_match = true;
+                    }
+                }
+                if let Some(expected) = if_modified_since {
+                    // NOTE: Equality comparison is good enough for tests.
+                    if last_modified != expected {
+                        all_match = false;
+                    } else {
+                        any_match = true;
+                    }
+                }
+
+                // Write out the main response line.
+                if any_match && all_match {
+                    buf.get_mut()
+                        .write_all(b"HTTP/1.1 304 Not Modified\r\n")
+                        .unwrap();
+                } else {
+                    buf.get_mut().write_all(b"HTTP/1.1 200 OK\r\n").unwrap();
+                }
+                // TODO: Support 451 for crate index deletions.
+
+                // Write out other headers.
+                buf.get_mut()
+                    .write_all(format!("Content-Length: {}\r\n", data.len()).as_bytes())
+                    .unwrap();
+                buf.get_mut()
+                    .write_all(format!("ETag: \"{}\"\r\n", etag).as_bytes())
+                    .unwrap();
+                buf.get_mut()
+                    .write_all(format!("Last-Modified: {}\r\n", last_modified).as_bytes())
+                    .unwrap();
+
+                // And finally, write out the body.
+                buf.get_mut().write_all(b"\r\n").unwrap();
+                buf.get_mut().write_all(&data).unwrap();
+            } else {
+                loop {
+                    line.clear();
+                    if buf.read_line(&mut line).unwrap() == 0 {
+                        // Connection terminated.
+                        continue 'server;
+                    }
+
+                    if line == "\r\n" {
+                        break;
+                    }
+                }
+
+                buf.get_mut()
+                    .write_all(b"HTTP/1.1 404 Not Found\r\n\r\n")
+                    .unwrap();
+                buf.get_mut().write_all(b"\r\n").unwrap();
+            }
+            buf.get_mut().flush().unwrap();
+        }
+    });
+
+    RegistryServer {
+        addr,
+        server: Some(t),
+        done,
+    }
+}
+
 pub fn init_registry(registry_path: PathBuf, dl_url: String, api_url: Url, api_path: PathBuf) {
     // Initialize a new registry.
     repo(&registry_path)

crates/resolver-tests/src/lib.rs

@@ -1,6 +1,7 @@
 #![allow(clippy::many_single_char_names)]
 #![allow(clippy::needless_range_loop)] // false positives
 
+use std::borrow::Cow;
 use std::cell::RefCell;
 use std::cmp::PartialEq;
 use std::cmp::{max, min};

src/bin/cargo/cli.rs

@@ -44,6 +44,7 @@ Available unstable (nightly-only) flags:
     -Z terminal-width      -- Provide a terminal width to rustc for error truncation
     -Z namespaced-features -- Allow features with `dep:` prefix
     -Z weak-dep-features   -- Allow `dep_name?/feature` feature syntax
+    -Z http-registries     -- Support HTTP-based crate registries
 
 Run with 'cargo -Z [FLAG] [SUBCOMMAND]'"
         );

src/cargo/core/dependency.rs

@@ -60,7 +60,7 @@ struct SerializedDependency<'a> {
     target: Option<&'a Platform>,
     /// The registry URL this dependency is from.
     /// If None, then it comes from the default registry (crates.io).
-    registry: Option<&'a str>,
+    registry: Option<String>,
 }
 
 impl ser::Serialize for Dependency {
@@ -79,7 +79,7 @@ impl ser::Serialize for Dependency {
             features: self.features(),
             target: self.platform(),
             rename: self.explicit_name_in_toml().map(|s| s.as_str()),
-            registry: registry_id.as_ref().map(|sid| sid.url().as_str()),
+            registry: registry_id.as_ref().map(|sid| sid.as_url().to_string()),
         }
         .serialize(s)
     }

src/cargo/core/features.rs

@@ -360,6 +360,7 @@ pub struct CliUnstable {
     pub namespaced_features: bool,
     pub weak_dep_features: bool,
     pub extra_link_arg: bool,
+    pub http_registry: bool,
 }
 
 fn deserialize_build_std<'de, D>(deserializer: D) -> Result<Option<Vec<String>>, D::Error>
@@ -468,6 +469,7 @@ impl CliUnstable {
             "namespaced-features" => self.namespaced_features = parse_empty(k, v)?,
             "weak-dep-features" => self.weak_dep_features = parse_empty(k, v)?,
             "extra-link-arg" => self.extra_link_arg = parse_empty(k, v)?,
+            "http-registry" => self.http_registry = parse_empty(k, v)?,
             _ => bail!("unknown `-Z` flag specified: {}", k),
         }
 

src/cargo/core/package.rs

@@ -8,12 +8,11 @@ use std::path::{Path, PathBuf};
 use std::rc::Rc;
 use std::time::{Duration, Instant};
 
-use anyhow::Context;
 use bytesize::ByteSize;
 use curl::easy::{Easy, HttpVersion};
 use curl::multi::{EasyHandle, Multi};
 use lazycell::LazyCell;
-use log::{debug, warn};
+use log::debug;
 use semver::Version;
 use serde::Serialize;
 
@@ -579,23 +578,6 @@ impl<'cfg> PackageSet<'cfg> {
     }
 }
 
-// When dynamically linked against libcurl, we want to ignore some failures
-// when using old versions that don't support certain features.
-macro_rules! try_old_curl {
-    ($e:expr, $msg:expr) => {
-        let result = $e;
-        if cfg!(target_os = "macos") {
-            if let Err(e) = result {
-                warn!("ignoring libcurl {} error: {}", $msg, e);
-            }
-        } else {
-            result.with_context(|| {
-                anyhow::format_err!("failed to enable {}, is curl not built right?", $msg)
-            })?;
-        }
-    };
-}
-
 impl<'a, 'cfg> Downloads<'a, 'cfg> {
     /// Starts to download the package for the `id` specified.
     ///

src/cargo/core/registry.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::collections::{HashMap, HashSet};
 
 use crate::core::PackageSet;
@@ -15,6 +16,14 @@ use url::Url;
 ///
 /// See also `core::Source`.
 pub trait Registry {
+    /// Give source the opportunity to batch pre-fetch dependency information.
+    fn prefetch(
+        &mut self,
+        _deps: &mut dyn Iterator<Item = Cow<'_, Dependency>>,
+    ) -> CargoResult<()> {
+        Ok(())
+    }
+
     /// Attempt to find the packages that match a dependency request.
     fn query(
         &mut self,
@@ -482,6 +491,58 @@ https://doc.rust-lang.org/cargo/reference/overriding-dependencies.html
 }
 
 impl<'cfg> Registry for PackageRegistry<'cfg> {
+    fn prefetch(&mut self, deps: &mut dyn Iterator<Item = Cow<'_, Dependency>>) -> CargoResult<()> {
+        assert!(self.patches_locked);
+
+        // We need to partition deps so that we can prefetch dependencies from different
+        // sources. Note that we do not prefetch from overrides.
+        let mut deps_per_source = HashMap::new();
+        for dep in deps {
+            // We need to check for patches, as they may tell us to look at a different source.
+            // If they do, we want to make sure we don't access the original registry
+            // unnecessarily.
+            let mut patches = Vec::<Summary>::new();
+            if let Some(extra) = self.patches.get(dep.source_id().canonical_url()) {
+                patches.extend(
+                    extra
+                        .iter()
+                        .filter(|s| dep.matches_ignoring_source(s.package_id()))
+                        .cloned(),
+                );
+            }
+
+            let source_id = if patches.len() == 1 && dep.is_locked() {
+                // Perform the prefetch from the patched-in source instead.
+                patches.remove(0).source_id()
+            } else {
+                // The code in `fn query` accesses the original source here, so we do too.
+                dep.source_id()
+            };
+
+            deps_per_source
+                .entry(source_id)
+                .or_insert_with(Vec::new)
+                .push(dep);
+        }
+
+        for (s, deps) in deps_per_source {
+            // Ensure the requested source_id is loaded
+            self.ensure_loaded(s, Kind::Normal).chain_err(|| {
+                anyhow::format_err!(
+                    "failed to load source for dependency `{}`",
+                    deps[0].package_name()
+                )
+            })?;
+
+            self.sources
+                .get_mut(s)
+                .unwrap()
+                .prefetch(&mut deps.into_iter())?;
+        }
+
+        Ok(())
+    }
+
     fn query(
         &mut self,
         dep: &Dependency,

src/cargo/core/resolver/mod.rs

@@ -47,6 +47,7 @@
 //! that we're implementing something that probably shouldn't be allocating all
 //! over the place.
 
+use std::borrow::Cow;
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::mem;
 use std::rc::Rc;
@@ -57,7 +58,7 @@ use log::{debug, trace};
 use crate::core::PackageIdSpec;
 use crate::core::{Dependency, PackageId, Registry, Summary};
 use crate::util::config::Config;
-use crate::util::errors::CargoResult;
+use crate::util::errors::{CargoResult, CargoResultExt};
 use crate::util::profile;
 
 use self::context::Context;
@@ -133,6 +134,16 @@ pub fn resolve(
         Some(config) => config.cli_unstable().minimal_versions,
         None => false,
     };
+
+    // First, allow the source to batch pre-fetch dependencies we may need.
+    registry
+        .prefetch(
+            &mut summaries
+                .iter()
+                .flat_map(|summary| summary.0.dependencies().iter().map(Cow::Borrowed)),
+        )
+        .chain_err(|| "failed to prefetch dependencies")?;
+
     let mut registry =
         RegistryQueryer::new(registry, replacements, try_to_use, minimal_versions, config);
     let cx = activate_deps_loop(cx, &mut registry, summaries, config)?;