changing vector name to record

Author: kubre

.idea/.gitignore

@@ -1,73 +1,73 @@
-package main
-
-import (
-	"encoding/csv"
-	"fmt"
-	"io"
-	"log"
-	"os"
-	"strconv"
-)
-
-func main() {
-	dataset := readCsvIntoSlice("dataset.csv")
-	questions := readCsvIntoSlice("questions.csv")
-
-	rankings := make([][]float64, len(questions))
-	for i, question := range questions {
-		temp := make([]float64, len(dataset))
-		for j, sentence := range dataset {
-			score := question.cosine_similarity(sentence)
-			temp[j] = score
-		}
-		rankings[i] = temp
-	}
-	fmt.Println(rankings)
-}
-
-func readCsvIntoSlice(name string) []Vector {
-	file, err := os.Open(name)
-	if err != nil {
-		log.Fatal("Cannot read the file", err)
-	}
-
-	r := csv.NewReader(file)
-	embeddings := []Vector{}
-
-	for {
-		record, err := r.Read()
-		if err == io.EOF {
-			break
-		}
-
-		tempEmbedding := convertStringListToFloats(record)
-
-		if err != nil {
-			log.Fatal("Error reading file", err)
-		}
-
-		embeddings = append(embeddings, tempEmbedding)
-	}
-
-	if err := file.Close(); err != nil {
-		log.Fatal("Not able to close the file")
-	}
-
-	return embeddings
-}
-
-func convertStringListToFloats(record []string) Vector {
-	tempEmbedding := Vector{
-		data: make([]float64, len(record)),
-	}
-
-	for i, value := range record {
-		point, err := strconv.ParseFloat(value, 64)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		tempEmbedding.data[i] = point
-	}
-	return tempEmbedding
-}
+package main
+
+import (
+	"encoding/csv"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"strconv"
+)
+
+func main() {
+	dataset := readCsvIntoSlice("dataset.csv")
+	questions := readCsvIntoSlice("questions.csv")
+
+	rankings := make([][]float64, len(questions))
+	for i, question := range questions {
+		temp := make([]float64, len(dataset))
+		for j, sentence := range dataset {
+			score := question.cosine_similarity(sentence)
+			temp[j] = score
+		}
+		rankings[i] = temp
+	}
+	fmt.Println(rankings)
+}
+
+func readCsvIntoSlice(name string) []Record {
+	file, err := os.Open(name)
+	if err != nil {
+		log.Fatal("Cannot read the file", err)
+	}
+
+	r := csv.NewReader(file)
+	embeddings := []Record{}
+
+	for {
+		record, err := r.Read()
+		if err == io.EOF {
+			break
+		}
+
+		tempEmbedding := convertStringListToFloats(record)
+
+		if err != nil {
+			log.Fatal("Error reading file", err)
+		}
+
+		embeddings = append(embeddings, tempEmbedding)
+	}
+
+	if err := file.Close(); err != nil {
+		log.Fatal("Not able to close the file")
+	}
+
+	return embeddings
+}
+
+func convertStringListToFloats(record []string) Record {
+	tempEmbedding := Record{
+		index: make([]float64, len(record)),
+	}
+
+	for i, value := range record {
+		point, err := strconv.ParseFloat(value, 64)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		tempEmbedding.index[i] = point
+	}
+	return tempEmbedding
+}

main.go

@@ -2,12 +2,26 @@
 
 Database for storing any querying the embeddings and metadata alongside it.
 
-
 ## Development Log (26 Aug 2023)
 
 **Requirements**
-- Querying
-    - Search on vector embeddings
-        - Cosine similarity
-- Storing
-    - Representing embeddings
+
+Indexing
+
+-   Flat Index
+    -   No modification to vectors store directly as series of floats
+-   How to build and index for embeddings
+    -   Random Projection
+        -   Using matrix of MxM size to convert N size vector to M size
+    -   HNSW (Hierarchical Navigable Small World)- Querying
+
+Querying
+
+-   Cosine similarity
+    -   Formula: dot(v, w) / (v.norm() \* w.norm())
+        -   dot v w: sum(vi \* wi)
+        -   norm v: sqrt(sum(vi \* vi))
+
+Storing
+
+-   How to store vector format

readme.md

@@ -0,0 +1,30 @@
+package main
+
+import "math"
+
+type Record struct {
+	index  []float64
+	normal float64
+}
+
+func (r Record) cosine_similarity(w Record) float64 {
+	return r.dot(w) / (r.norm() * w.norm())
+}
+
+func (r Record) dot(w Record) float64 {
+	sum := 0.0
+	for i := 0; i < len(r.index); i++ {
+		sum += r.index[i] * w.index[i]
+	}
+	return sum
+}
+
+func (r Record) norm() float64 {
+	sum := 0.0
+	for _, elem := range r.index {
+		sum += elem * elem
+	}
+
+	// TODO: Need to check how much perf gain is to return cached version
+	return math.Sqrt(sum)
+}

record.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestDotProductFor2Vectors(t *testing.T) {
+	v := Record{
+		index: []float64{1, 2, 3},
+	}
+	w := Record{
+		index: []float64{4, 5, 6},
+	}
+
+	dotProduct := v.dot(w)
+
+	if dotProduct != 32 {
+		t.Fatal("Dot product of", v.index, "and", w.index, "should be 3")
+	}
+}
+
+func TestEmptyVectors(t *testing.T) {
+	v := Record{}
+	w := Record{}
+
+	dotProduct := v.dot(w)
+
+	if dotProduct != 0 {
+		t.Fatal("Dot product of", v.index, "and", w.index, "should be 3")
+	}
+}
+
+func TestNormAVector(t *testing.T) {
+	v := Record{
+		index: []float64{1, 2, 3},
+	}
+
+	normal := v.norm()
+
+	if normal != 3.7416573867739413 {
+		t.Fatal("Norm of vector", v.index, "should be", 3.7416573867739413, "but it is", normal)
+	}
+}