From 336451340b86ba1f713b47d44225df61058f5a8f Mon Sep 17 00:00:00 2001
From: Grail Finder <wohilas@gmail.com>
Date: Wed, 29 Jan 2025 20:18:40 +0300
Subject: Feat: set/change props from tui for /completion

---
 README.md        |  8 ++++----
 bot.go           |  2 ++
 llm.go           |  2 +-
 models/models.go | 17 +++++++++++------
 tui.go           |  1 -
 5 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index 769fb7c..8e9db0f 100644
--- a/README.md
+++ b/README.md
@@ -36,11 +36,11 @@
 - boolean flag to use/not use tools. I see it as a msg from a tool to an llm "Hey, it might be good idea to use me!";
 - connection to a model status;
 - ===== /llamacpp specific (it has a different body -> interface instead of global var)
-- edit syscards / create new ones;
+- edit syscards; +
 - consider adding use /completion of llamacpp, since openai endpoint clearly has template|format issues; +
-- change temp, min-p and other params from tui;
+- change temp, min-p and other params from tui; +
 - DRY; +
-- keybind to switch between openai and llamacpp endpoints;
+- keybind to switch between openai and llamacpp endpoints (chat vs completion);
 - option to remove <thinking> from chat history;
 - in chat management table add preview of the last message; +
 
@@ -66,6 +66,6 @@
 - number of sentences in a batch should depend on number of words there. +
 - F1 can load any chat, by loading chat of other agent it does not switch agents, if that chat is continued, it will rewrite agent in db; (either allow only chats from current agent OR switch agent on chat loading); +
 - after chat is deleted: load undeleted chat; +
-- name split for llamacpp completion. user msg should end with 'bot_name:';
+- name split for llamacpp completion. user msg should end with 'bot_name:'; +
 - add retry on failed call (and EOF);
 - model info shold be an event and show disconnect status when fails;
diff --git a/bot.go b/bot.go
index 753b112..c256f58 100644
--- a/bot.go
+++ b/bot.go
@@ -41,6 +41,8 @@ var (
 	defaultLCPProps     = map[string]float32{
 		"temperature":    0.8,
 		"dry_multiplier": 0.6,
+		"min_p":          0.05,
+		"n_predict":      -1.0,
 	}
 )
 
diff --git a/llm.go b/llm.go
index 3fb248a..d847da1 100644
--- a/llm.go
+++ b/llm.go
@@ -52,7 +52,7 @@ func (lcp LlamaCPPeer) FormMsg(msg, role string) (io.Reader, error) {
 	}
 	prompt := strings.Join(messages, "\n")
 	botMsgStart := "\n" + cfg.AssistantRole + ":\n"
-	payload := models.NewLCPReq(prompt+botMsgStart, role)
+	payload := models.NewLCPReq(prompt+botMsgStart, role, defaultLCPProps)
 	data, err := json.Marshal(payload)
 	if err != nil {
 		logger.Error("failed to form a msg", "error", err)
diff --git a/models/models.go b/models/models.go
index 526d056..6404464 100644
--- a/models/models.go
+++ b/models/models.go
@@ -157,6 +157,8 @@ type LlamaCPPReq struct {
 	Temperature   float32  `json:"temperature"`
 	DryMultiplier float32  `json:"dry_multiplier"`
 	Stop          []string `json:"stop"`
+	MinP          float32  `json:"min_p"`
+	NPredict      int32    `json:"n_predict"`
 	// MaxTokens        int     `json:"max_tokens"`
 	// DryBase          float64 `json:"dry_base"`
 	// DryAllowedLength int     `json:"dry_allowed_length"`
@@ -166,7 +168,6 @@ type LlamaCPPReq struct {
 	// DynatempExponent int     `json:"dynatemp_exponent"`
 	// TopK             int     `json:"top_k"`
 	// TopP             float32 `json:"top_p"`
-	// MinP             float32 `json:"min_p"`
 	// TypicalP         int     `json:"typical_p"`
 	// XtcProbability   int     `json:"xtc_probability"`
 	// XtcThreshold     float32 `json:"xtc_threshold"`
@@ -177,12 +178,16 @@ type LlamaCPPReq struct {
 	// Samplers         string  `json:"samplers"`
 }
 
-func NewLCPReq(prompt, role string) LlamaCPPReq {
+func NewLCPReq(prompt, role string, props map[string]float32) LlamaCPPReq {
 	return LlamaCPPReq{
-		Stream:        true,
-		Prompt:        prompt,
-		Temperature:   0.8,
-		DryMultiplier: 0.5,
+		Stream: true,
+		Prompt: prompt,
+		// Temperature:   0.8,
+		// DryMultiplier: 0.5,
+		Temperature:   props["temperature"],
+		DryMultiplier: props["dry_multiplier"],
+		MinP:          props["min_p"],
+		NPredict:      int32(props["n_predict"]),
 		Stop:          []string{role + ":\n", "<|im_end|>"},
 	}
 }
diff --git a/tui.go b/tui.go
index 6766cd5..53d1c10 100644
--- a/tui.go
+++ b/tui.go
@@ -135,7 +135,6 @@ func startNewChat() {
 func makePropsForm(props map[string]float32) *tview.Form {
 	form := tview.NewForm().
 		AddTextView("Notes", "Props for llamacpp completion call", 40, 2, true, false).
-		AddCheckbox("Age 18+", false, nil).
 		AddButton("Quit", func() {
 			pages.RemovePage(propsPage)
 		})
-- 
cgit v1.2.3