Feat: preload lcp model

2 files changed, 63 insertions, 1 deletions

diff --git a/bot.go b/bot.go
index e2f03b8..8a0ba0a 100644
--- a/bot.go
+++ b/bot.go
@@ -16,6 +16,7 @@ import (
 	"log/slog"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"strings"
@@ -188,6 +189,58 @@ func createClient(connectTimeout time.Duration) *http.Client {
 	}
 }
 
+func warmUpModel() {
+	u, err := url.Parse(cfg.CurrentAPI)
+	if err != nil {
+		return
+	}
+	host := u.Hostname()
+	if host != "localhost" && host != "127.0.0.1" && host != "::1" {
+		return
+	}
+	go func() {
+		var data []byte
+		var err error
+		if strings.HasSuffix(cfg.CurrentAPI, "/completion") {
+			// Old completion endpoint
+			req := models.NewLCPReq(".", chatBody.Model, nil, map[string]float32{
+				"temperature":    0.8,
+				"dry_multiplier": 0.0,
+				"min_p":          0.05,
+				"n_predict":      0,
+			}, []string{})
+			req.Stream = false
+			data, err = json.Marshal(req)
+		} else if strings.Contains(cfg.CurrentAPI, "/v1/chat/completions") {
+			// OpenAI-compatible chat endpoint
+			req := models.OpenAIReq{
+				ChatBody: &models.ChatBody{
+					Model: chatBody.Model,
+					Messages: []models.RoleMsg{
+						{Role: "system", Content: "."},
+					},
+					Stream: false,
+				},
+				Tools: nil,
+			}
+			data, err = json.Marshal(req)
+		} else {
+			// Unknown local endpoint, skip
+			return
+		}
+		if err != nil {
+			logger.Debug("failed to marshal warmup request", "error", err)
+			return
+		}
+		resp, err := httpClient.Post(cfg.CurrentAPI, "application/json", bytes.NewReader(data))
+		if err != nil {
+			logger.Debug("warmup request failed", "error", err)
+			return
+		}
+		resp.Body.Close()
+	}()
+}
+
 func fetchLCPModelName() *models.LCPModels {
 	//nolint
 	resp, err := httpClient.Get(cfg.FetchModelNameAPI)
@@ -894,7 +947,7 @@ func init() {
 		cluedoState = extra.CluedoPrepCards(playerOrder)
 	}
 	choseChunkParser()
-	httpClient = createClient(time.Second * 15)
+	httpClient = createClient(time.Second * 90)
 	if cfg.TTS_ENABLED {
 		orator = extra.NewOrator(logger, cfg)
 	}
diff --git a/tui.go b/tui.go
index 941371b..53c8cfd 100644
--- a/tui.go
+++ b/tui.go
@@ -89,6 +89,7 @@ var (
 [yellow]Alt+4[white]: edit msg role
 [yellow]Alt+5[white]: toggle system and tool messages display
 [yellow]Alt+6[white]: toggle status line visibility
+[yellow]Alt+9[white]: warm up (load) selected llama.cpp model
 
 === scrolling chat window (some keys similar to vim) ===
 [yellow]arrows up/down and j/k[white]: scroll up and down
@@ -1235,6 +1236,14 @@ func init() {
 			toggleShellMode()
 			return nil
 		}
+		if event.Key() == tcell.KeyRune && event.Modifiers() == tcell.ModAlt && event.Rune() == '9' {
+			// Warm up (load) the currently selected model
+			go warmUpModel()
+			if err := notifyUser("model warmup", "loading model: "+chatBody.Model); err != nil {
+				logger.Debug("failed to notify user", "error", err)
+			}
+			return nil
+		}
 		// cannot send msg in editMode or botRespMode
 		if event.Key() == tcell.KeyEscape && !editMode && !botRespMode {
 			msgText := textArea.GetText()