From 7bbedd93cf078fc7496a6779cf9eda6e588e64c0 Mon Sep 17 00:00:00 2001 From: Grail Finder Date: Tue, 7 Jan 2025 11:11:44 +0300 Subject: Enha (RAG): raw text as primary key in vector db --- storage/migrations/002_add_vector.up.sql | 6 ++---- storage/vector.go | 11 ++++------- 2 files changed, 6 insertions(+), 11 deletions(-) (limited to 'storage') diff --git a/storage/migrations/002_add_vector.up.sql b/storage/migrations/002_add_vector.up.sql index f64aecb..6c8fb52 100644 --- a/storage/migrations/002_add_vector.up.sql +++ b/storage/migrations/002_add_vector.up.sql @@ -1,13 +1,11 @@ CREATE VIRTUAL TABLE IF NOT EXISTS embeddings USING vec0( - id INTEGER PRIMARY KEY AUTOINCREMENT, embedding FLOAT[5120], slug TEXT NOT NULL, - raw_text TEXT NOT NULL + raw_text TEXT PRIMARY KEY, ); CREATE VIRTUAL TABLE IF NOT EXISTS embeddings_384 USING vec0( - id INTEGER PRIMARY KEY AUTOINCREMENT, embedding FLOAT[384], slug TEXT NOT NULL, - raw_text TEXT NOT NULL + raw_text TEXT PRIMARY KEY ); diff --git a/storage/vector.go b/storage/vector.go index 1579686..fe479d8 100644 --- a/storage/vector.go +++ b/storage/vector.go @@ -68,7 +68,6 @@ func (p ProviderSQL) WriteVector(row *models.VectorRow) error { } err = stmt.Exec() if err != nil { - p.logger.Error("failed exec a stmt", "error", err) return err } return nil @@ -85,7 +84,6 @@ func (p ProviderSQL) SearchClosest(q []float32) ([]models.VectorRow, error) { } stmt, _, err := p.s3Conn.Prepare( fmt.Sprintf(`SELECT - id, distance, embedding, slug, @@ -109,12 +107,11 @@ func (p ProviderSQL) SearchClosest(q []float32) ([]models.VectorRow, error) { resp := []models.VectorRow{} for stmt.Step() { res := models.VectorRow{} - res.ID = uint32(stmt.ColumnInt64(0)) - res.Distance = float32(stmt.ColumnFloat(1)) - emb := stmt.ColumnRawText(2) + res.Distance = float32(stmt.ColumnFloat(0)) + emb := stmt.ColumnRawText(1) res.Embeddings = decodeUnsafe(emb) - res.Slug = stmt.ColumnText(3) - res.RawText = stmt.ColumnText(4) + res.Slug = stmt.ColumnText(2) + res.RawText = stmt.ColumnText(3) resp = append(resp, res) } if err := stmt.Err(); err != nil { -- cgit v1.2.3