Resurrect CBOR index handling and migration, update to migrate failed builds

.cargo/audit.toml

@@ -3,6 +3,9 @@ ignore = [
     "RUSTSEC-2020-0036", # failure is officially deprecated/unmaintained
     # https://github.com/rust-lang/docs.rs/issues/1014
 
+    "RUSTSEC-2021-0127", # serde_cbor is unmaintained
+    # https://github.com/rust-lang/docs.rs/issues/1568
+
     "RUSTSEC-2023-0018", # rustwide -> remove_dir_all,TOCTOU / Race Condition 
     # https://github.com/rust-lang/docs.rs/issues/2074
 ]

Cargo.toml

@@ -68,6 +68,7 @@ string_cache = "0.8.0"
 postgres-types = { version = "0.2", features = ["derive"] }
 zip = {version = "0.6.2", default-features = false, features = ["bzip2"]}
 bzip2 = "0.4.4"
+serde_cbor = "0.11.1"
 getrandom = "0.2.1"
 itertools = { version = "0.11.0", optional = true}
 rusqlite = { version = "0.29.0", features = ["bundled"] }
@@ -86,6 +87,7 @@ uuid = "1.1.2"
 # Data serialization and deserialization
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+memmap2 = "0.5.0"
 
 # axum dependencies
 axum = { version = "0.6.1", features = ["headers"]}

src/bin/cratesfyi.rs

@@ -15,8 +15,8 @@ use docs_rs::utils::{
     remove_crate_priority, set_crate_priority, ConfigName,
 };
 use docs_rs::{
-    start_background_metrics_webserver, start_web_server, BuildQueue, Config, Context, Index,
-    InstanceMetrics, PackageKind, RustwideBuilder, ServiceMetrics, Storage,
+    migrate_old_archive_indexes, start_background_metrics_webserver, start_web_server, BuildQueue,
+    Config, Context, Index, InstanceMetrics, PackageKind, RustwideBuilder, ServiceMetrics, Storage,
 };
 use humantime::Duration;
 use once_cell::sync::OnceCell;
@@ -482,6 +482,9 @@ enum DatabaseSubcommand {
     /// Backfill GitHub/Gitlab stats for crates.
     BackfillRepositoryStats,
 
+    /// migrate the old CBOR archive index files to SQLIte
+    MigrateArchiveIndex,
+
     /// Updates info for a crate from the registry's API
     UpdateCrateRegistryFields {
         #[arg(name = "CRATE")]
@@ -533,6 +536,10 @@ impl DatabaseSubcommand {
                 ctx.repository_stats_updater()?.update_all_crates()?;
             }
 
+            Self::MigrateArchiveIndex => {
+                migrate_old_archive_indexes(&*ctx.storage()?, &mut *ctx.conn()?)?;
+            }
+
             Self::BackfillRepositoryStats => {
                 ctx.repository_stats_updater()?.backfill_repositories()?;
             }

src/lib.rs

@@ -9,6 +9,7 @@ pub use self::docbuilder::PackageKind;
 pub use self::docbuilder::RustwideBuilder;
 pub use self::index::Index;
 pub use self::metrics::{InstanceMetrics, ServiceMetrics};
+pub use self::storage::migrate_old_archive_indexes;
 pub use self::storage::Storage;
 pub use self::web::{start_background_metrics_webserver, start_web_server};