Chore: remove old rag

3 files changed, 0 insertions, 658 deletions

diff --git a/rag_new/embedder.go b/rag_new/embedder.go
deleted file mode 100644
index 27b975a..0000000
--- a/rag_new/embedder.go
+++ /dev/null
@@ -1,98 +0,0 @@
-package rag_new
-
-import (
-	"bytes"
-	"gf-lt/config"
-	"encoding/json"
-	"fmt"
-	"log/slog"
-	"net/http"
-)
-
-// Embedder defines the interface for embedding text
-type Embedder interface {
-	Embed(text []string) ([][]float32, error)
-	EmbedSingle(text string) ([]float32, error)
-}
-
-// APIEmbedder implements embedder using an API (like Hugging Face, OpenAI, etc.)
-type APIEmbedder struct {
-	logger  *slog.Logger
-	client  *http.Client
-	cfg     *config.Config
-}
-
-func NewAPIEmbedder(l *slog.Logger, cfg *config.Config) *APIEmbedder {
-	return &APIEmbedder{
-		logger: l,
-		client: &http.Client{},
-		cfg:    cfg,
-	}
-}
-
-func (a *APIEmbedder) Embed(text []string) ([][]float32, error) {
-	payload, err := json.Marshal(
-		map[string]any{"inputs": text, "options": map[string]bool{"wait_for_model": true}},
-	)
-	if err != nil {
-		a.logger.Error("failed to marshal payload", "err", err.Error())
-		return nil, err
-	}
-
-	req, err := http.NewRequest("POST", a.cfg.EmbedURL, bytes.NewReader(payload))
-	if err != nil {
-		a.logger.Error("failed to create new req", "err", err.Error())
-		return nil, err
-	}
-	
-	if a.cfg.HFToken != "" {
-		req.Header.Add("Authorization", "Bearer "+a.cfg.HFToken)
-	}
-	
-	resp, err := a.client.Do(req)
-	if err != nil {
-		a.logger.Error("failed to embed text", "err", err.Error())
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		err = fmt.Errorf("non 200 response; code: %v", resp.StatusCode)
-		a.logger.Error(err.Error())
-		return nil, err
-	}
-
-	var emb [][]float32
-	if err := json.NewDecoder(resp.Body).Decode(&emb); err != nil {
-		a.logger.Error("failed to decode embedding response", "err", err.Error())
-		return nil, err
-	}
-
-	if len(emb) == 0 {
-		err = fmt.Errorf("empty embedding response")
-		a.logger.Error("empty embedding response")
-		return nil, err
-	}
-
-	return emb, nil
-}
-
-func (a *APIEmbedder) EmbedSingle(text string) ([]float32, error) {
-	result, err := a.Embed([]string{text})
-	if err != nil {
-		return nil, err
-	}
-	if len(result) == 0 {
-		return nil, fmt.Errorf("no embeddings returned")
-	}
-	return result[0], nil
-}
-
-// TODO: ONNXEmbedder implementation would go here
-// This would require:
-// 1. Loading ONNX models locally
-// 2. Using a Go ONNX runtime (like gorgonia/onnx or similar)
-// 3. Converting text to embeddings without external API calls
-//
-// For now, we'll focus on the API implementation which is already working in the current system,
-// and can be extended later when we have ONNX runtime integration
\ No newline at end of file
diff --git a/rag_new/rag.go b/rag_new/rag.go
deleted file mode 100644
index d012087..0000000
--- a/rag_new/rag.go
+++ /dev/null
@@ -1,260 +0,0 @@
-package rag_new
-
-import (
-	"gf-lt/config"
-	"gf-lt/models"
-	"gf-lt/storage"
-	"fmt"
-	"log/slog"
-	"os"
-	"path"
-	"strings"
-	"sync"
-
-	"github.com/neurosnap/sentences/english"
-)
-
-var (
-	// Status messages for TUI integration
-	LongJobStatusCh = make(chan string, 10) // Increased buffer size to prevent blocking
-	FinishedRAGStatus   = "finished loading RAG file; press Enter"
-	LoadedFileRAGStatus = "loaded file"
-	ErrRAGStatus        = "some error occurred; failed to transfer data to vector db"
-)
-
-type RAG struct {
-	logger *slog.Logger
-	store  storage.FullRepo
-	cfg    *config.Config
-	embedder Embedder
-	storage *VectorStorage
-}
-
-func New(l *slog.Logger, s storage.FullRepo, cfg *config.Config) *RAG {
-	// Initialize with API embedder by default, could be configurable later
-	embedder := NewAPIEmbedder(l, cfg)
-	
-	rag := &RAG{
-		logger: l,
-		store:  s,
-		cfg:    cfg,
-		embedder: embedder,
-		storage: NewVectorStorage(l, s),
-	}
-	
-	// Create the necessary tables
-	if err := rag.storage.CreateTables(); err != nil {
-		l.Error("failed to create vector tables", "error", err)
-	}
-	
-	return rag
-}
-
-func wordCounter(sentence string) int {
-	return len(strings.Split(strings.TrimSpace(sentence), " "))
-}
-
-func (r *RAG) LoadRAG(fpath string) error {
-	data, err := os.ReadFile(fpath)
-	if err != nil {
-		return err
-	}
-	r.logger.Debug("rag: loaded file", "fp", fpath)
-	LongJobStatusCh <- LoadedFileRAGStatus
-	
-	fileText := string(data)
-	tokenizer, err := english.NewSentenceTokenizer(nil)
-	if err != nil {
-		return err
-	}
-	sentences := tokenizer.Tokenize(fileText)
-	sents := make([]string, len(sentences))
-	for i, s := range sentences {
-		sents[i] = s.Text
-	}
-	
-	// Group sentences into paragraphs based on word limit
-	paragraphs := []string{}
-	par := strings.Builder{}
-	for i := 0; i < len(sents); i++ {
-		// Only add sentences that aren't empty
-		if strings.TrimSpace(sents[i]) != "" {
-			if par.Len() > 0 {
-				par.WriteString(" ") // Add space between sentences
-			}
-			par.WriteString(sents[i])
-		}
-		
-		if wordCounter(par.String()) > int(r.cfg.RAGWordLimit) {
-			paragraph := strings.TrimSpace(par.String())
-			if paragraph != "" {
-				paragraphs = append(paragraphs, paragraph)
-			}
-			par.Reset()
-		}
-	}
-	
-	// Handle any remaining content in the paragraph buffer
-	if par.Len() > 0 {
-		paragraph := strings.TrimSpace(par.String())
-		if paragraph != "" {
-			paragraphs = append(paragraphs, paragraph)
-		}
-	}
-	
-	// Adjust batch size if needed
-	if len(paragraphs) < int(r.cfg.RAGBatchSize) && len(paragraphs) > 0 {
-		r.cfg.RAGBatchSize = len(paragraphs)
-	}
-	
-	if len(paragraphs) == 0 {
-		return fmt.Errorf("no valid paragraphs found in file")
-	}
-	
-	var (
-		maxChSize = 100
-		left      = 0
-		right     = r.cfg.RAGBatchSize
-		batchCh   = make(chan map[int][]string, maxChSize)
-		vectorCh  = make(chan []models.VectorRow, maxChSize)
-		errCh     = make(chan error, 1)
-		doneCh    = make(chan bool, 1)
-		lock      = new(sync.Mutex)
-	)
-	
-	defer close(doneCh)
-	defer close(errCh)
-	defer close(batchCh)
-	
-	// Fill input channel with batches
-	ctn := 0
-	totalParagraphs := len(paragraphs)
-	for {
-		if int(right) > totalParagraphs {
-			batchCh <- map[int][]string{left: paragraphs[left:]}
-			break
-		}
-		batchCh <- map[int][]string{left: paragraphs[left:right]}
-		left, right = right, right+r.cfg.RAGBatchSize
-		ctn++
-	}
-	
-	finishedBatchesMsg := fmt.Sprintf("finished batching batches#: %d; paragraphs: %d; sentences: %d\n", ctn+1, len(paragraphs), len(sents))
-	r.logger.Debug(finishedBatchesMsg)
-	LongJobStatusCh <- finishedBatchesMsg
-	
-	// Start worker goroutines
-	for w := 0; w < int(r.cfg.RAGWorkers); w++ {
-		go r.batchToVectorAsync(lock, w, batchCh, vectorCh, errCh, doneCh, path.Base(fpath))
-	}
-	
-	// Wait for embedding to be done
-	<-doneCh
-	
-	// Write vectors to storage
-	return r.writeVectors(vectorCh)
-}
-
-func (r *RAG) writeVectors(vectorCh chan []models.VectorRow) error {
-	for {
-		for batch := range vectorCh {
-			for _, vector := range batch {
-				if err := r.storage.WriteVector(&vector); err != nil {
-					r.logger.Error("failed to write vector", "error", err, "slug", vector.Slug)
-					LongJobStatusCh <- ErrRAGStatus
-					continue // a duplicate is not critical
-				}
-			}
-			r.logger.Debug("wrote batch to db", "size", len(batch), "vector_chan_len", len(vectorCh))
-			if len(vectorCh) == 0 {
-				r.logger.Debug("finished writing vectors")
-				LongJobStatusCh <- FinishedRAGStatus
-				return nil
-			}
-		}
-	}
-}
-
-func (r *RAG) batchToVectorAsync(lock *sync.Mutex, id int, inputCh <-chan map[int][]string,
-	vectorCh chan<- []models.VectorRow, errCh chan error, doneCh chan bool, filename string) {
-	defer func() {
-		if len(doneCh) == 0 {
-			doneCh <- true
-		}
-	}()
-	
-	for {
-		lock.Lock()
-		if len(inputCh) == 0 {
-			lock.Unlock()
-			return
-		}
-		
-		select {
-		case linesMap := <-inputCh:
-			for leftI, lines := range linesMap {
-				if err := r.fetchEmb(lines, errCh, vectorCh, fmt.Sprintf("%s_%d", filename, leftI), filename); err != nil {
-					r.logger.Error("error fetching embeddings", "error", err, "worker", id)
-					lock.Unlock()
-					return
-				}
-			}
-			lock.Unlock()
-		case err := <-errCh:
-			r.logger.Error("got an error from error channel", "error", err)
-			lock.Unlock()
-			return
-		default:
-			lock.Unlock()
-		}
-		
-		r.logger.Debug("processed batch", "batches#", len(inputCh), "worker#", id)
-		LongJobStatusCh <- fmt.Sprintf("converted to vector; batches: %d, worker#: %d", len(inputCh), id)
-	}
-}
-
-func (r *RAG) fetchEmb(lines []string, errCh chan error, vectorCh chan<- []models.VectorRow, slug, filename string) error {
-	embeddings, err := r.embedder.Embed(lines)
-	if err != nil {
-		r.logger.Error("failed to embed lines", "err", err.Error())
-		errCh <- err
-		return err
-	}
-	
-	if len(embeddings) == 0 {
-		err := fmt.Errorf("no embeddings returned")
-		r.logger.Error("empty embeddings")
-		errCh <- err
-		return err
-	}
-	
-	vectors := make([]models.VectorRow, len(embeddings))
-	for i, emb := range embeddings {
-		vector := models.VectorRow{
-			Embeddings: emb,
-			RawText:    lines[i],
-			Slug:       fmt.Sprintf("%s_%d", slug, i),
-			FileName:   filename,
-		}
-		vectors[i] = vector
-	}
-	
-	vectorCh <- vectors
-	return nil
-}
-
-func (r *RAG) LineToVector(line string) ([]float32, error) {
-	return r.embedder.EmbedSingle(line)
-}
-
-func (r *RAG) SearchEmb(emb *models.EmbeddingResp) ([]models.VectorRow, error) {
-	return r.storage.SearchClosest(emb.Embedding)
-}
-
-func (r *RAG) ListLoaded() ([]string, error) {
-	return r.storage.ListFiles()
-}
-
-func (r *RAG) RemoveFile(filename string) error {
-	return r.storage.RemoveEmbByFileName(filename)
-}
\ No newline at end of file
diff --git a/rag_new/storage.go b/rag_new/storage.go
deleted file mode 100644
index 2ab56fb..0000000
--- a/rag_new/storage.go
+++ /dev/null
@@ -1,300 +0,0 @@
-package rag_new
-
-import (
-	"gf-lt/models"
-	"gf-lt/storage"
-	"encoding/binary"
-	"fmt"
-	"log/slog"
-	"sort"
-	"strings"
-	"unsafe"
-
-	"github.com/jmoiron/sqlx"
-)
-
-// VectorStorage handles storing and retrieving vectors from SQLite
-type VectorStorage struct {
-	logger *slog.Logger
-	sqlxDB *sqlx.DB
-	store  storage.FullRepo
-}
-
-func NewVectorStorage(logger *slog.Logger, store storage.FullRepo) *VectorStorage {
-	return &VectorStorage{
-		logger: logger,
-		sqlxDB: store.DB(),  // Use the new DB() method
-		store:  store,
-	}
-}
-
-// CreateTables creates the necessary tables for vector storage
-func (vs *VectorStorage) CreateTables() error {
-	// Create tables for different embedding dimensions
-	queries := []string{
-		`CREATE TABLE IF NOT EXISTS embeddings_384 (
-			id INTEGER PRIMARY KEY AUTOINCREMENT,
-			embeddings BLOB NOT NULL,
-			slug TEXT NOT NULL,
-			raw_text TEXT NOT NULL,
-			filename TEXT NOT NULL,
-			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-		)`,
-		`CREATE TABLE IF NOT EXISTS embeddings_5120 (
-			id INTEGER PRIMARY KEY AUTOINCREMENT,
-			embeddings BLOB NOT NULL,
-			slug TEXT NOT NULL,
-			raw_text TEXT NOT NULL,
-			filename TEXT NOT NULL,
-			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-		)`,
-		// Indexes for better performance
-		`CREATE INDEX IF NOT EXISTS idx_embeddings_384_filename ON embeddings_384(filename)`,
-		`CREATE INDEX IF NOT EXISTS idx_embeddings_5120_filename ON embeddings_5120(filename)`,
-		`CREATE INDEX IF NOT EXISTS idx_embeddings_384_slug ON embeddings_384(slug)`,
-		`CREATE INDEX IF NOT EXISTS idx_embeddings_5120_slug ON embeddings_5120(slug)`,
-		
-		// Additional indexes that may help with searches
-		`CREATE INDEX IF NOT EXISTS idx_embeddings_384_created_at ON embeddings_384(created_at)`,
-		`CREATE INDEX IF NOT EXISTS idx_embeddings_5120_created_at ON embeddings_5120(created_at)`,
-	}
-
-	for _, query := range queries {
-		if _, err := vs.sqlxDB.Exec(query); err != nil {
-			return fmt.Errorf("failed to create table: %w", err)
-		}
-	}
-	return nil
-}
-
-// SerializeVector converts []float32 to binary blob
-func SerializeVector(vec []float32) []byte {
-	buf := make([]byte, len(vec)*4) // 4 bytes per float32
-	for i, v := range vec {
-		binary.LittleEndian.PutUint32(buf[i*4:], mathFloat32bits(v))
-	}
-	return buf
-}
-
-// DeserializeVector converts binary blob back to []float32
-func DeserializeVector(data []byte) []float32 {
-	count := len(data) / 4
-	vec := make([]float32, count)
-	for i := 0; i < count; i++ {
-		vec[i] = mathBitsToFloat32(binary.LittleEndian.Uint32(data[i*4:]))
-	}
-	return vec
-}
-
-// mathFloat32bits and mathBitsToFloat32 are helpers to convert between float32 and uint32
-func mathFloat32bits(f float32) uint32 {
-	return binary.LittleEndian.Uint32((*(*[4]byte)(unsafe.Pointer(&f)))[:4])
-}
-
-func mathBitsToFloat32(b uint32) float32 {
-	return *(*float32)(unsafe.Pointer(&b))
-}
-
-// WriteVector stores an embedding vector in the database
-func (vs *VectorStorage) WriteVector(row *models.VectorRow) error {
-	tableName, err := vs.getTableName(row.Embeddings)
-	if err != nil {
-		return err
-	}
-
-	// Serialize the embeddings to binary
-	serializedEmbeddings := SerializeVector(row.Embeddings)
-
-	query := fmt.Sprintf(
-		"INSERT INTO %s (embeddings, slug, raw_text, filename) VALUES (?, ?, ?, ?)",
-		tableName,
-	)
-
-	if _, err := vs.sqlxDB.Exec(query, serializedEmbeddings, row.Slug, row.RawText, row.FileName); err != nil {
-		vs.logger.Error("failed to write vector", "error", err, "slug", row.Slug)
-		return err
-	}
-
-	return nil
-}
-
-// getTableName determines which table to use based on embedding size
-func (vs *VectorStorage) getTableName(emb []float32) (string, error) {
-	switch len(emb) {
-	case 384:
-		return "embeddings_384", nil
-	case 5120:
-		return "embeddings_5120", nil
-	default:
-		return "", fmt.Errorf("no table for embedding size of %d", len(emb))
-	}
-}
-
-// SearchClosest finds vectors closest to the query vector using efficient cosine similarity calculation
-func (vs *VectorStorage) SearchClosest(query []float32) ([]models.VectorRow, error) {
-	tableName, err := vs.getTableName(query)
-	if err != nil {
-		return nil, err
-	}
-
-	// For better performance, instead of loading all vectors at once,
-	// we'll implement batching and potentially add L2 distance-based pre-filtering
-	// since cosine similarity is related to L2 distance for normalized vectors
-	
-	querySQL := fmt.Sprintf("SELECT embeddings, slug, raw_text, filename FROM %s", tableName)
-	rows, err := vs.sqlxDB.Query(querySQL)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	// Use a min-heap or simple slice to keep track of top 3 closest vectors
-	type SearchResult struct {
-		vector   models.VectorRow
-		distance float32
-	}
-	
-	var topResults []SearchResult
-	
-	// Process vectors one by one to avoid loading everything into memory
-	for rows.Next() {
-		var (
-			embeddingsBlob []byte
-			slug, rawText, fileName string
-		)
-		
-		if err := rows.Scan(&embeddingsBlob, &slug, &rawText, &fileName); err != nil {
-			vs.logger.Error("failed to scan row", "error", err)
-			continue
-		}
-		
-		storedEmbeddings := DeserializeVector(embeddingsBlob)
-		
-		// Calculate cosine similarity (returns value between -1 and 1, where 1 is most similar)
-		similarity := cosineSimilarity(query, storedEmbeddings)
-		distance := 1 - similarity // Convert to distance where 0 is most similar
-		
-		result := SearchResult{
-			vector: models.VectorRow{
-				Embeddings: storedEmbeddings,
-				Slug:       slug,
-				RawText:    rawText,
-				FileName:   fileName,
-			},
-			distance: distance,
-		}
-		
-		// Add to top results and maintain only top 3
-		topResults = append(topResults, result)
-		
-		// Sort and keep only top 3
-		sort.Slice(topResults, func(i, j int) bool {
-			return topResults[i].distance < topResults[j].distance
-		})
-		
-		if len(topResults) > 3 {
-			topResults = topResults[:3] // Keep only closest 3
-		}
-	}
-	
-	// Convert back to VectorRow slice
-	var results []models.VectorRow
-	for _, result := range topResults {
-		result.vector.Distance = result.distance
-		results = append(results, result.vector)
-	}
-	
-	return results, nil
-}
-
-// ListFiles returns a list of all loaded files
-func (vs *VectorStorage) ListFiles() ([]string, error) {
-	var fileLists [][]string
-	
-	// Query both tables and combine results
-	for _, table := range []string{"embeddings_384", "embeddings_5120"} {
-		query := fmt.Sprintf("SELECT DISTINCT filename FROM %s", table)
-		rows, err := vs.sqlxDB.Query(query)
-		if err != nil {
-			// Continue if one table doesn't exist
-			continue
-		}
-		
-		var files []string
-		for rows.Next() {
-			var filename string
-			if err := rows.Scan(&filename); err != nil {
-				continue
-			}
-			files = append(files, filename)
-		}
-		rows.Close()
-		
-		fileLists = append(fileLists, files)
-	}
-	
-	// Combine and deduplicate
-	fileSet := make(map[string]bool)
-	var allFiles []string
-	for _, files := range fileLists {
-		for _, file := range files {
-			if !fileSet[file] {
-				fileSet[file] = true
-				allFiles = append(allFiles, file)
-			}
-		}
-	}
-	
-	return allFiles, nil
-}
-
-// RemoveEmbByFileName removes all embeddings associated with a specific filename
-func (vs *VectorStorage) RemoveEmbByFileName(filename string) error {
-	var errors []string
-	
-	for _, table := range []string{"embeddings_384", "embeddings_5120"} {
-		query := fmt.Sprintf("DELETE FROM %s WHERE filename = ?", table)
-		if _, err := vs.sqlxDB.Exec(query, filename); err != nil {
-			errors = append(errors, err.Error())
-		}
-	}
-	
-	if len(errors) > 0 {
-		return fmt.Errorf("errors occurred: %s", strings.Join(errors, "; "))
-	}
-	
-	return nil
-}
-
-// cosineSimilarity calculates the cosine similarity between two vectors
-func cosineSimilarity(a, b []float32) float32 {
-	if len(a) != len(b) {
-		return 0.0
-	}
-
-	var dotProduct, normA, normB float32
-	for i := 0; i < len(a); i++ {
-		dotProduct += a[i] * b[i]
-		normA += a[i] * a[i]
-		normB += b[i] * b[i]
-	}
-
-	if normA == 0 || normB == 0 {
-		return 0.0
-	}
-
-	return dotProduct / (sqrt(normA) * sqrt(normB))
-}
-
-// sqrt returns the square root of a float32
-func sqrt(f float32) float32 {
-	// A simple implementation of square root using Newton's method
-	if f == 0 {
-		return 0
-	}
-	guess := f / 2
-	for i := 0; i < 10; i++ { // 10 iterations should be enough for good precision
-		guess = (guess + f/guess) / 2
-	}
-	return guess
-}
\ No newline at end of file