diff options
| -rw-r--r-- | README.md | 8 | ||||
| -rw-r--r-- | bot.go | 2 | ||||
| -rw-r--r-- | llm.go | 2 | ||||
| -rw-r--r-- | models/models.go | 17 | ||||
| -rw-r--r-- | tui.go | 1 |
5 files changed, 18 insertions, 12 deletions
@@ -36,11 +36,11 @@ - boolean flag to use/not use tools. I see it as a msg from a tool to an llm "Hey, it might be good idea to use me!"; - connection to a model status; - ===== /llamacpp specific (it has a different body -> interface instead of global var) -- edit syscards / create new ones; +- edit syscards; + - consider adding use /completion of llamacpp, since openai endpoint clearly has template|format issues; + -- change temp, min-p and other params from tui; +- change temp, min-p and other params from tui; + - DRY; + -- keybind to switch between openai and llamacpp endpoints; +- keybind to switch between openai and llamacpp endpoints (chat vs completion); - option to remove <thinking> from chat history; - in chat management table add preview of the last message; + @@ -66,6 +66,6 @@ - number of sentences in a batch should depend on number of words there. + - F1 can load any chat, by loading chat of other agent it does not switch agents, if that chat is continued, it will rewrite agent in db; (either allow only chats from current agent OR switch agent on chat loading); + - after chat is deleted: load undeleted chat; + -- name split for llamacpp completion. user msg should end with 'bot_name:'; +- name split for llamacpp completion. user msg should end with 'bot_name:'; + - add retry on failed call (and EOF); - model info shold be an event and show disconnect status when fails; @@ -41,6 +41,8 @@ var ( defaultLCPProps = map[string]float32{ "temperature": 0.8, "dry_multiplier": 0.6, + "min_p": 0.05, + "n_predict": -1.0, } ) @@ -52,7 +52,7 @@ func (lcp LlamaCPPeer) FormMsg(msg, role string) (io.Reader, error) { } prompt := strings.Join(messages, "\n") botMsgStart := "\n" + cfg.AssistantRole + ":\n" - payload := models.NewLCPReq(prompt+botMsgStart, role) + payload := models.NewLCPReq(prompt+botMsgStart, role, defaultLCPProps) data, err := json.Marshal(payload) if err != nil { logger.Error("failed to form a msg", "error", err) diff --git a/models/models.go b/models/models.go index 526d056..6404464 100644 --- a/models/models.go +++ b/models/models.go @@ -157,6 +157,8 @@ type LlamaCPPReq struct { Temperature float32 `json:"temperature"` DryMultiplier float32 `json:"dry_multiplier"` Stop []string `json:"stop"` + MinP float32 `json:"min_p"` + NPredict int32 `json:"n_predict"` // MaxTokens int `json:"max_tokens"` // DryBase float64 `json:"dry_base"` // DryAllowedLength int `json:"dry_allowed_length"` @@ -166,7 +168,6 @@ type LlamaCPPReq struct { // DynatempExponent int `json:"dynatemp_exponent"` // TopK int `json:"top_k"` // TopP float32 `json:"top_p"` - // MinP float32 `json:"min_p"` // TypicalP int `json:"typical_p"` // XtcProbability int `json:"xtc_probability"` // XtcThreshold float32 `json:"xtc_threshold"` @@ -177,12 +178,16 @@ type LlamaCPPReq struct { // Samplers string `json:"samplers"` } -func NewLCPReq(prompt, role string) LlamaCPPReq { +func NewLCPReq(prompt, role string, props map[string]float32) LlamaCPPReq { return LlamaCPPReq{ - Stream: true, - Prompt: prompt, - Temperature: 0.8, - DryMultiplier: 0.5, + Stream: true, + Prompt: prompt, + // Temperature: 0.8, + // DryMultiplier: 0.5, + Temperature: props["temperature"], + DryMultiplier: props["dry_multiplier"], + MinP: props["min_p"], + NPredict: int32(props["n_predict"]), Stop: []string{role + ":\n", "<|im_end|>"}, } } @@ -135,7 +135,6 @@ func startNewChat() { func makePropsForm(props map[string]float32) *tview.Form { form := tview.NewForm(). AddTextView("Notes", "Props for llamacpp completion call", 40, 2, true, false). - AddCheckbox("Age 18+", false, nil). AddButton("Quit", func() { pages.RemovePage(propsPage) }) |