Don't check out the crates.io index locally

src/cargo/sources/registry/index.rs

@@ -1,14 +1,15 @@
 use std::collections::HashMap;
-use std::io::prelude::*;
-use std::fs::File;
 use std::path::Path;
+use std::str;
 
 use serde_json;
 
 use core::dependency::{Dependency, DependencyInner, Kind};
 use core::{SourceId, Summary, PackageId, Registry};
 use sources::registry::{RegistryPackage, RegistryDependency, INDEX_LOCK};
+use sources::registry::RegistryData;
 use util::{CargoResult, ChainError, internal, Filesystem, Config};
+use util::human;
 
 pub struct RegistryIndex<'cfg> {
     source_id: SourceId,
@@ -23,7 +24,8 @@ impl<'cfg> RegistryIndex<'cfg> {
     pub fn new(id: &SourceId,
                path: &Filesystem,
                config: &'cfg Config,
-               locked: bool) -> RegistryIndex<'cfg> {
+               locked: bool)
+               -> RegistryIndex<'cfg> {
         RegistryIndex {
             source_id: id.clone(),
             path: path.clone(),
@@ -35,13 +37,16 @@ impl<'cfg> RegistryIndex<'cfg> {
     }
 
     /// Return the hash listed for a specified PackageId.
-    pub fn hash(&mut self, pkg: &PackageId) -> CargoResult<String> {
+    pub fn hash(&mut self,
+                pkg: &PackageId,
+                load: &mut RegistryData)
+                -> CargoResult<String> {
         let key = (pkg.name().to_string(), pkg.version().to_string());
         if let Some(s) = self.hashes.get(&key) {
             return Ok(s.clone())
         }
         // Ok, we're missing the key, so parse the index file to load it.
-        self.summaries(pkg.name())?;
+        self.summaries(pkg.name(), load)?;
         self.hashes.get(&key).chain_error(|| {
             internal(format!("no hash listed for {}", pkg))
         }).map(|s| s.clone())
@@ -51,20 +56,26 @@ impl<'cfg> RegistryIndex<'cfg> {
     ///
     /// Returns a list of pairs of (summary, yanked) for the package name
     /// specified.
-    pub fn summaries(&mut self, name: &str) -> CargoResult<&Vec<(Summary, bool)>> {
+    pub fn summaries(&mut self,
+                     name: &str,
+                     load: &mut RegistryData)
+                     -> CargoResult<&Vec<(Summary, bool)>> {
         if self.cache.contains_key(name) {
             return Ok(&self.cache[name]);
         }
-        let summaries = self.load_summaries(name)?;
+        let summaries = self.load_summaries(name, load)?;
         let summaries = summaries.into_iter().filter(|summary| {
             summary.0.package_id().name() == name
         }).collect();
         self.cache.insert(name.to_string(), summaries);
         Ok(&self.cache[name])
     }
 
-    fn load_summaries(&mut self, name: &str) -> CargoResult<Vec<(Summary, bool)>> {
-        let (path, _lock) = if self.locked {
+    fn load_summaries(&mut self,
+                      name: &str,
+                      load: &mut RegistryData)
+                      -> CargoResult<Vec<(Summary, bool)>> {
+        let (root, _lock) = if self.locked {
             let lock = self.path.open_ro(Path::new(INDEX_LOCK),
                                          self.config,
                                          "the registry index");
@@ -84,25 +95,32 @@ impl<'cfg> RegistryIndex<'cfg> {
 
         // see module comment for why this is structured the way it is
         let path = match fs_name.len() {
-            1 => path.join("1").join(&fs_name),
-            2 => path.join("2").join(&fs_name),
-            3 => path.join("3").join(&fs_name[..1]).join(&fs_name),
-            _ => path.join(&fs_name[0..2])
-                     .join(&fs_name[2..4])
-                     .join(&fs_name),
+            1 => format!("1/{}", fs_name),
+            2 => format!("2/{}", fs_name),
+            3 => format!("3/{}/{}", &fs_name[..1], fs_name),
+            _ => format!("{}/{}/{}", &fs_name[0..2], &fs_name[2..4], fs_name),
+            // 1 => Path::new("1").join(fs_name),
+            // 2 => Path::new("2").join(fs_name),
+            // 3 => Path::new("3").join(&fs_name[..1]).join(fs_name),
+            // _ => Path::new(&fs_name[0..2]).join(&fs_name[2..4]).join(fs_name),
         };
-        match File::open(&path) {
-            Ok(mut f) => {
-                let mut contents = String::new();
-                f.read_to_string(&mut contents)?;
-                let ret: CargoResult<Vec<(Summary, bool)>>;
-                ret = contents.lines().filter(|l| !l.trim().is_empty())
-                              .map(|l| self.parse_registry_package(l))
-                              .collect();
-                ret.chain_error(|| {
-                    internal(format!("failed to parse registry's information \
-                                      for: {}", name))
-                })
+        match load.load(&root, Path::new(&path)) {
+            Ok(contents) => {
+                let contents = str::from_utf8(&contents).map_err(|_| {
+                    human("registry index file was not valid utf-8")
+                })?;
+                let lines = contents.lines()
+                                    .map(|s| s.trim())
+                                    .filter(|l| !l.is_empty());
+
+                // Attempt forwards-compatibility on the index by ignoring
+                // everything that we ourselves don't understand, that should
+                // allow future cargo implementations to break the
+                // interpretation of each line here and older cargo will simply
+                // ignore the new lines.
+                Ok(lines.filter_map(|line| {
+                    self.parse_registry_package(line).ok()
+                }).collect())
             }
             Err(..) => Ok(Vec::new()),
         }
@@ -161,12 +179,13 @@ impl<'cfg> RegistryIndex<'cfg> {
               .set_kind(kind)
               .into_dependency())
     }
-}
 
-impl<'cfg> Registry for RegistryIndex<'cfg> {
-    fn query(&mut self, dep: &Dependency) -> CargoResult<Vec<Summary>> {
+    pub fn query(&mut self,
+                 dep: &Dependency,
+                 load: &mut RegistryData)
+                 -> CargoResult<Vec<Summary>> {
         let mut summaries = {
-            let summaries = self.summaries(dep.name())?;
+            let summaries = self.summaries(dep.name(), load)?;
             summaries.iter().filter(|&&(_, yanked)| {
                 dep.source_id().precise().is_some() || !yanked
             }).map(|s| s.0.clone()).collect::<Vec<_>>()
@@ -188,8 +207,4 @@ impl<'cfg> Registry for RegistryIndex<'cfg> {
         });
         summaries.query(dep)
     }
-
-    fn supports_checksums(&self) -> bool {
-        true
-    }
 }

src/cargo/sources/registry/local.rs

@@ -6,8 +6,9 @@ use rustc_serialize::hex::ToHex;
 
 use core::PackageId;
 use sources::registry::{RegistryData, RegistryConfig};
-use util::{Config, CargoResult, ChainError, human, Sha256, Filesystem};
 use util::FileLock;
+use util::paths;
+use util::{Config, CargoResult, ChainError, human, Sha256, Filesystem};
 
 pub struct LocalRegistry<'cfg> {
     index_path: Filesystem,
@@ -34,7 +35,11 @@ impl<'cfg> RegistryData for LocalRegistry<'cfg> {
         &self.index_path
     }
 
-    fn config(&self) -> CargoResult<Option<RegistryConfig>> {
+    fn load(&self, root: &Path, path: &Path) -> CargoResult<Vec<u8>> {
+        paths::read_bytes(&root.join(path))
+    }
+
+    fn config(&mut self) -> CargoResult<Option<RegistryConfig>> {
         // Local registries don't have configuration for remote APIs or anything
         // like that
         Ok(None)

src/cargo/sources/registry/mod.rs

@@ -219,7 +219,8 @@ struct RegistryDependency {
 
 pub trait RegistryData {
     fn index_path(&self) -> &Filesystem;
-    fn config(&self) -> CargoResult<Option<RegistryConfig>>;
+    fn load(&self, root: &Path, path: &Path) -> CargoResult<Vec<u8>>;
+    fn config(&mut self) -> CargoResult<Option<RegistryConfig>>;
     fn update_index(&mut self) -> CargoResult<()>;
     fn download(&mut self,
                 pkg: &PackageId,
@@ -274,7 +275,7 @@ impl<'cfg> RegistrySource<'cfg> {
     /// Decode the configuration stored within the registry.
     ///
     /// This requires that the index has been at least checked out.
-    pub fn config(&self) -> CargoResult<Option<RegistryConfig>> {
+    pub fn config(&mut self) -> CargoResult<Option<RegistryConfig>> {
         self.ops.config()
     }
 
@@ -323,12 +324,12 @@ impl<'cfg> Registry for RegistrySource<'cfg> {
         // come back with no summaries, then our registry may need to be
         // updated, so we fall back to performing a lazy update.
         if dep.source_id().precise().is_some() && !self.updated {
-            if self.index.query(dep)?.is_empty() {
+            if self.index.query(dep, &mut *self.ops)?.is_empty() {
                 self.do_update()?;
             }
         }
 
-        self.index.query(dep)
+        self.index.query(dep, &mut *self.ops)
     }
 
     fn supports_checksums(&self) -> bool {
@@ -356,7 +357,7 @@ impl<'cfg> Source for RegistrySource<'cfg> {
     }
 
     fn download(&mut self, package: &PackageId) -> CargoResult<Package> {
-        let hash = self.index.hash(package)?;
+        let hash = self.index.hash(package, &mut *self.ops)?;
         let path = self.ops.download(package, &hash)?;
         let path = self.unpack_package(package, &path).chain_error(|| {
             internal(format!("failed to unpack package `{}`", package))
@@ -369,7 +370,7 @@ impl<'cfg> Source for RegistrySource<'cfg> {
         // differ due to historical Cargo bugs. To paper over these we trash the
         // *summary* loaded from the Cargo.toml we just downloaded with the one
         // we loaded from the index.
-        let summaries = self.index.summaries(package.name())?;
+        let summaries = self.index.summaries(package.name(), &mut *self.ops)?;
         let summary = summaries.iter().map(|s| &s.0).find(|s| {
             s.package_id() == package
         }).expect("summary not found");