refactor: move algorithm to a crate

Signed-off-by: usamoi <[email protected]>

.taplo.toml

@@ -2,9 +2,16 @@
 indent_string = "    "
 
 [[rule]]
-keys = ["dependencies", "*-denpendencies", "lints", "patch.*", "profile.*"]
-
-[rule.formatting]
-reorder_keys = true
-reorder_arrays = true
-align_comments = true
+include = ["**/Cargo.toml"]
+keys = [
+    "dependencies",
+    "dev-dependencies",
+    "build-dependencies",
+    "target.*.dependencies",
+    "lints",
+    "patch.*",
+    "profile.*",
+    "workspace.dependencies",
+    "lints.dependencies",
+]
+formatting = { reorder_keys = true, reorder_arrays = true, align_comments = true }

Cargo.toml

@@ -20,23 +20,23 @@ pg16 = ["pgrx/pg16", "pgrx-catalog/pg16"]
 pg17 = ["pgrx/pg17", "pgrx-catalog/pg17"]
 
 [dependencies]
+algorithm = { path = "./crates/algorithm" }
 always_equal = { path = "./crates/always_equal" }
 distance = { path = "./crates/distance" }
+k_means = { path = "./crates/k_means" }
 rabitq = { path = "./crates/rabitq" }
 random_orthogonal_matrix = { path = "./crates/random_orthogonal_matrix" }
 simd = { path = "./crates/simd" }
 vector = { path = "./crates/vector" }
 
 half.workspace = true
-log = "0.4.25"
 paste = "1"
 pgrx = { version = "=0.12.9", default-features = false, features = ["cshim"] }
 pgrx-catalog = "0.1.0"
 rand.workspace = true
-rayon = "1.10.0"
 serde.workspace = true
 toml = "0.8.19"
-validator = { version = "0.19.0", features = ["derive"] }
+validator.workspace = true
 zerocopy = "0.8.14"
 zerocopy-derive = "0.8.14"
 
@@ -56,14 +56,17 @@ edition = "2021"
 
 [workspace.dependencies]
 half = { version = "2.4.1", features = ["serde", "zerocopy"] }
+log = "0.4.25"
 rand = "0.8.5"
 serde = "1"
+validator = { version = "0.20.0", features = ["derive"] }
 
 [workspace.lints]
 clippy.identity_op = "allow"
 clippy.int_plus_one = "allow"
 clippy.needless_range_loop = "allow"
 clippy.nonminimal_bool = "allow"
+rust.unsafe_code = "deny"
 rust.unsafe_op_in_unsafe_fn = "deny"
 rust.unused_lifetimes = "warn"
 rust.unused_qualifications = "warn"

crates/algorithm/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "algorithm"
+version.workspace = true
+edition.workspace = true
+
+[dependencies]
+always_equal = { path = "../always_equal" }
+distance = { path = "../distance" }
+k_means = { path = "../k_means" }
+rabitq = { path = "../rabitq" }
+random_orthogonal_matrix = { path = "../random_orthogonal_matrix" }
+simd = { path = "../simd" }
+vector = { path = "../vector" }
+
+half.workspace = true
+paste = "1"
+rand.workspace = true
+serde.workspace = true
+toml = "0.8.19"
+validator.workspace = true
+zerocopy = "0.8.14"
+zerocopy-derive = "0.8.14"
+
+[lints]
+workspace = true

crates/algorithm/src/build.rs

@@ -0,0 +1,101 @@
+use crate::RelationWrite;
+use crate::operator::{Accessor2, Operator, Vector};
+use crate::tape::*;
+use crate::tuples::*;
+use crate::types::*;
+use vector::VectorOwned;
+
+pub fn build<O: Operator>(
+    vector_options: VectorOptions,
+    vchordrq_options: VchordrqIndexingOptions,
+    index: impl RelationWrite,
+    structures: Vec<Structure<O::Vector>>,
+) {
+    let dims = vector_options.dims;
+    let is_residual = vchordrq_options.residual_quantization && O::SUPPORTS_RESIDUAL;
+    let mut meta = TapeWriter::<_, _, MetaTuple>::create(|| index.extend(false));
+    assert_eq!(meta.first(), 0);
+    let freepage = TapeWriter::<_, _, FreepageTuple>::create(|| index.extend(false));
+    let mut vectors = TapeWriter::<_, _, VectorTuple<O::Vector>>::create(|| index.extend(true));
+    let mut pointer_of_means = Vec::<Vec<IndexPointer>>::new();
+    for i in 0..structures.len() {
+        let mut level = Vec::new();
+        for j in 0..structures[i].len() {
+            let vector = structures[i].means[j].as_borrowed();
+            let (metadata, slices) = O::Vector::vector_split(vector);
+            let mut chain = Ok(metadata);
+            for i in (0..slices.len()).rev() {
+                chain = Err(vectors.push(match chain {
+                    Ok(metadata) => VectorTuple::_0 {
+                        payload: None,
+                        elements: slices[i].to_vec(),
+                        metadata,
+                    },
+                    Err(pointer) => VectorTuple::_1 {
+                        payload: None,
+                        elements: slices[i].to_vec(),
+                        pointer,
+                    },
+                }));
+            }
+            level.push(chain.err().unwrap());
+        }
+        pointer_of_means.push(level);
+    }
+    let mut pointer_of_firsts = Vec::<Vec<u32>>::new();
+    for i in 0..structures.len() {
+        let mut level = Vec::new();
+        for j in 0..structures[i].len() {
+            if i == 0 {
+                let tape = TapeWriter::<_, _, H0Tuple>::create(|| index.extend(false));
+                let mut jump = TapeWriter::<_, _, JumpTuple>::create(|| index.extend(false));
+                jump.push(JumpTuple {
+                    first: tape.first(),
+                });
+                level.push(jump.first());
+            } else {
+                let mut tape = H1TapeWriter::<_, _>::create(|| index.extend(false));
+                let h2_mean = structures[i].means[j].as_borrowed();
+                let h2_children = structures[i].children[j].as_slice();
+                for child in h2_children.iter().copied() {
+                    let h1_mean = structures[i - 1].means[child as usize].as_borrowed();
+                    let code = if is_residual {
+                        let mut residual_accessor = O::ResidualAccessor::default();
+                        residual_accessor.push(
+                            O::Vector::elements_and_metadata(h1_mean).0,
+                            O::Vector::elements_and_metadata(h2_mean).0,
+                        );
+                        let residual = residual_accessor.finish(
+                            O::Vector::elements_and_metadata(h1_mean).1,
+                            O::Vector::elements_and_metadata(h2_mean).1,
+                        );
+                        O::Vector::code(residual.as_borrowed())
+                    } else {
+                        O::Vector::code(h1_mean)
+                    };
+                    tape.push(H1Branch {
+                        mean: pointer_of_means[i - 1][child as usize],
+                        dis_u_2: code.dis_u_2,
+                        factor_ppc: code.factor_ppc,
+                        factor_ip: code.factor_ip,
+                        factor_err: code.factor_err,
+                        signs: code.signs,
+                        first: pointer_of_firsts[i - 1][child as usize],
+                    });
+                }
+                let tape = tape.into_inner();
+                level.push(tape.first());
+            }
+        }
+        pointer_of_firsts.push(level);
+    }
+    meta.push(MetaTuple {
+        dims,
+        height_of_root: structures.len() as u32,
+        is_residual,
+        vectors_first: vectors.first(),
+        root_mean: pointer_of_means.last().unwrap()[0],
+        root_first: pointer_of_firsts.last().unwrap()[0],
+        freepage_first: freepage.first(),
+    });
+}