diff options
Diffstat (limited to 'models')
| -rw-r--r-- | models/card.go | 30 | ||||
| -rw-r--r-- | models/consts.go | 13 | ||||
| -rw-r--r-- | models/db.go | 2 | ||||
| -rw-r--r-- | models/deepseek.go | 144 | ||||
| -rw-r--r-- | models/embed.go | 15 | ||||
| -rw-r--r-- | models/extra.go | 49 | ||||
| -rw-r--r-- | models/models.go | 587 | ||||
| -rw-r--r-- | models/openrouter.go | 187 | ||||
| -rw-r--r-- | models/openrouter_test.go | 96 |
9 files changed, 1044 insertions, 79 deletions
diff --git a/models/card.go b/models/card.go index adfb030..0bf437c 100644 --- a/models/card.go +++ b/models/card.go @@ -1,6 +1,10 @@ package models -import "strings" +import ( + "crypto/md5" + "fmt" + "strings" +) // https://github.com/malfoyslastname/character-card-spec-v2/blob/main/spec_v2.md // what a bloat; trim to Role->Msg pair and first msg @@ -31,18 +35,26 @@ func (c *CharCardSpec) Simplify(userName, fpath string) *CharCard { fm := strings.ReplaceAll(strings.ReplaceAll(c.FirstMes, "{{char}}", c.Name), "{{user}}", userName) sysPr := strings.ReplaceAll(strings.ReplaceAll(c.Description, "{{char}}", c.Name), "{{user}}", userName) return &CharCard{ - SysPrompt: sysPr, - FirstMsg: fm, - Role: c.Name, - FilePath: fpath, + ID: ComputeCardID(c.Name, fpath), + SysPrompt: sysPr, + FirstMsg: fm, + Role: c.Name, + FilePath: fpath, + Characters: []string{c.Name, userName}, } } +func ComputeCardID(role, filePath string) string { + return fmt.Sprintf("%x", md5.Sum([]byte(role+filePath))) +} + type CharCard struct { - SysPrompt string `json:"sys_prompt"` - FirstMsg string `json:"first_msg"` - Role string `json:"role"` - FilePath string `json:"filepath"` + ID string `json:"id"` + SysPrompt string `json:"sys_prompt"` + FirstMsg string `json:"first_msg"` + Role string `json:"role"` + Characters []string `json:"chars"` + FilePath string `json:"filepath"` } func (cc *CharCard) ToSpec(userName string) *CharCardSpec { diff --git a/models/consts.go b/models/consts.go new file mode 100644 index 0000000..8b4002b --- /dev/null +++ b/models/consts.go @@ -0,0 +1,13 @@ +package models + +const ( + LoadedMark = "(loaded) " + ToolRespMultyType = "multimodel_content" +) + +type APIType int + +const ( + APITypeChat APIType = iota + APITypeCompletion +) diff --git a/models/db.go b/models/db.go index 090f46d..73a0b53 100644 --- a/models/db.go +++ b/models/db.go @@ -14,7 +14,7 @@ type Chat struct { UpdatedAt time.Time `db:"updated_at" json:"updated_at"` } -func (c Chat) ToHistory() ([]RoleMsg, error) { +func (c *Chat) ToHistory() ([]RoleMsg, error) { resp := []RoleMsg{} if err := json.Unmarshal([]byte(c.Msgs), &resp); err != nil { return nil, err diff --git a/models/deepseek.go b/models/deepseek.go new file mode 100644 index 0000000..8f9868d --- /dev/null +++ b/models/deepseek.go @@ -0,0 +1,144 @@ +package models + +type DSChatReq struct { + Messages []RoleMsg `json:"messages"` + Model string `json:"model"` + Stream bool `json:"stream"` + FrequencyPenalty int `json:"frequency_penalty"` + MaxTokens int `json:"max_tokens"` + PresencePenalty int `json:"presence_penalty"` + Temperature float32 `json:"temperature"` + TopP float32 `json:"top_p"` + // ResponseFormat struct { + // Type string `json:"type"` + // } `json:"response_format"` + // Stop any `json:"stop"` + // StreamOptions any `json:"stream_options"` + // Tools any `json:"tools"` + // ToolChoice string `json:"tool_choice"` + // Logprobs bool `json:"logprobs"` + // TopLogprobs any `json:"top_logprobs"` +} + +func NewDSChatReq(cb ChatBody) DSChatReq { + return DSChatReq{ + Messages: cb.Messages, + Model: cb.Model, + Stream: cb.Stream, + MaxTokens: 2048, + PresencePenalty: 0, + FrequencyPenalty: 0, + Temperature: 1.0, + TopP: 1.0, + } +} + +type DSCompletionReq struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + Echo bool `json:"echo"` + FrequencyPenalty int `json:"frequency_penalty"` + // Logprobs int `json:"logprobs"` + MaxTokens int `json:"max_tokens"` + PresencePenalty int `json:"presence_penalty"` + Stop any `json:"stop"` + Stream bool `json:"stream"` + StreamOptions any `json:"stream_options"` + Suffix any `json:"suffix"` + Temperature float32 `json:"temperature"` + TopP float32 `json:"top_p"` +} + +func NewDSCompletionReq(prompt, model string, temp float32, stopSlice []string) DSCompletionReq { + return DSCompletionReq{ + Model: model, + Prompt: prompt, + Temperature: temp, + Stream: true, + Echo: false, + MaxTokens: 2048, + PresencePenalty: 0, + FrequencyPenalty: 0, + TopP: 1.0, + Stop: stopSlice, + } +} + +type DSCompletionResp struct { + ID string `json:"id"` + Choices []struct { + FinishReason string `json:"finish_reason"` + Index int `json:"index"` + Logprobs struct { + TextOffset []int `json:"text_offset"` + TokenLogprobs []int `json:"token_logprobs"` + Tokens []string `json:"tokens"` + TopLogprobs []struct { + } `json:"top_logprobs"` + } `json:"logprobs"` + Text string `json:"text"` + } `json:"choices"` + Created int `json:"created"` + Model string `json:"model"` + SystemFingerprint string `json:"system_fingerprint"` + Object string `json:"object"` + Usage struct { + CompletionTokens int `json:"completion_tokens"` + PromptTokens int `json:"prompt_tokens"` + PromptCacheHitTokens int `json:"prompt_cache_hit_tokens"` + PromptCacheMissTokens int `json:"prompt_cache_miss_tokens"` + TotalTokens int `json:"total_tokens"` + CompletionTokensDetails struct { + ReasoningTokens int `json:"reasoning_tokens"` + } `json:"completion_tokens_details"` + } `json:"usage"` +} + +type DSChatResp struct { + Choices []struct { + Delta struct { + Content string `json:"content"` + Role any `json:"role"` + } `json:"delta"` + FinishReason string `json:"finish_reason"` + Index int `json:"index"` + Logprobs any `json:"logprobs"` + } `json:"choices"` + Created int `json:"created"` + ID string `json:"id"` + Model string `json:"model"` + Object string `json:"object"` + SystemFingerprint string `json:"system_fingerprint"` + Usage struct { + CompletionTokens int `json:"completion_tokens"` + PromptTokens int `json:"prompt_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage"` +} + +type DSChatStreamResp struct { + ID string `json:"id"` + Object string `json:"object"` + Created int `json:"created"` + Model string `json:"model"` + SystemFingerprint string `json:"system_fingerprint"` + Choices []struct { + Index int `json:"index"` + Delta struct { + Content string `json:"content"` + ReasoningContent string `json:"reasoning_content"` + } `json:"delta"` + Logprobs any `json:"logprobs"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` +} + +type DSBalance struct { + IsAvailable bool `json:"is_available"` + BalanceInfos []struct { + Currency string `json:"currency"` + TotalBalance string `json:"total_balance"` + GrantedBalance string `json:"granted_balance"` + ToppedUpBalance string `json:"topped_up_balance"` + } `json:"balance_infos"` +} diff --git a/models/embed.go b/models/embed.go new file mode 100644 index 0000000..078312c --- /dev/null +++ b/models/embed.go @@ -0,0 +1,15 @@ +package models + +type LCPEmbedResp struct { + Model string `json:"model"` + Object string `json:"object"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage"` + Data []struct { + Embedding []float32 `json:"embedding"` + Index int `json:"index"` + Object string `json:"object"` + } `json:"data"` +} diff --git a/models/extra.go b/models/extra.go new file mode 100644 index 0000000..5c60a26 --- /dev/null +++ b/models/extra.go @@ -0,0 +1,49 @@ +package models + +import ( + "regexp" + "strings" +) + +type AudioFormat string + +const ( + AFWav AudioFormat = "wav" + AFMP3 AudioFormat = "mp3" +) + +var threeOrMoreDashesRE = regexp.MustCompile(`-{3,}`) + +// CleanText removes markdown and special characters that are not suitable for TTS +func CleanText(text string) string { + // Remove markdown-like characters that might interfere with TTS + text = strings.ReplaceAll(text, "*", "") // Bold/italic markers + text = strings.ReplaceAll(text, "#", "") // Headers + text = strings.ReplaceAll(text, "_", "") // Underline/italic markers + text = strings.ReplaceAll(text, "~", "") // Strikethrough markers + text = strings.ReplaceAll(text, "`", "") // Code markers + text = strings.ReplaceAll(text, "[", "") // Link brackets + text = strings.ReplaceAll(text, "]", "") // Link brackets + text = strings.ReplaceAll(text, "!", "") // Exclamation marks (if not punctuation) + // Remove HTML tags using regex + htmlTagRegex := regexp.MustCompile(`<[^>]*>`) + text = htmlTagRegex.ReplaceAllString(text, "") + // Split text into lines to handle table separators + lines := strings.Split(text, "\n") + var filteredLines []string + for _, line := range lines { + // Check if the line looks like a table separator (e.g., |----|, |===|, | - - - |) + // A table separator typically contains only |, -, =, and spaces + isTableSeparator := regexp.MustCompile(`^\s*\|\s*[-=\s]+\|\s*$`).MatchString(strings.TrimSpace(line)) + if !isTableSeparator { + // If it's not a table separator, remove vertical bars but keep the content + processedLine := strings.ReplaceAll(line, "|", "") + filteredLines = append(filteredLines, processedLine) + } + // If it is a table separator, skip it (don't add to filteredLines) + } + text = strings.Join(filteredLines, "\n") + text = threeOrMoreDashesRE.ReplaceAllString(text, "") + text = strings.TrimSpace(text) // Remove leading/trailing whitespace + return text +} diff --git a/models/models.go b/models/models.go index bb61abf..97d0272 100644 --- a/models/models.go +++ b/models/models.go @@ -1,14 +1,23 @@ package models import ( - "elefant/config" + "encoding/base64" + "encoding/json" "fmt" + "os" "strings" ) type FuncCall struct { - Name string `json:"name"` - Args []string `json:"args"` + ID string `json:"id,omitempty"` + Name string `json:"name"` + Args map[string]string `json:"args"` +} + +type ToolCall struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + Args string `json:"arguments"` } type LLMResp struct { @@ -31,13 +40,26 @@ type LLMResp struct { ID string `json:"id"` } +type ToolDeltaFunc struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +type ToolDeltaResp struct { + ID string `json:"id,omitempty"` + Index int `json:"index"` + Function ToolDeltaFunc `json:"function"` +} + // for streaming type LLMRespChunk struct { Choices []struct { FinishReason string `json:"finish_reason"` Index int `json:"index"` Delta struct { - Content string `json:"content"` + Content string `json:"content"` + ReasoningContent string `json:"reasoning_content"` + ToolCalls []ToolDeltaResp `json:"tool_calls"` } `json:"delta"` } `json:"choices"` Created int `json:"created"` @@ -51,23 +73,344 @@ type LLMRespChunk struct { } `json:"usage"` } +type TextChunk struct { + Chunk string + ToolChunk string + Finished bool + ToolResp bool + FuncName string + ToolID string + Reasoning string // For models that send reasoning separately (OpenRouter, etc.) +} + +type TextContentPart struct { + Type string `json:"type"` + Text string `json:"text"` +} + +type ImageContentPart struct { + Type string `json:"type"` + Path string `json:"path,omitempty"` // Store original file path + ImageURL struct { + URL string `json:"url"` + } `json:"image_url"` +} + +// RoleMsg represents a message with content that can be either a simple string or structured content parts type RoleMsg struct { - Role string `json:"role"` - Content string `json:"content"` + Role string `json:"role"` + Content string `json:"-"` + ContentParts []any `json:"-"` + ToolCallID string `json:"tool_call_id,omitempty"` // For tool response messages + ToolCall *ToolCall `json:"tool_call,omitempty"` // For assistant messages with tool calls + IsShellCommand bool `json:"is_shell_command,omitempty"` // True for shell command outputs (always shown) + KnownTo []string `json:"known_to,omitempty"` + Stats *ResponseStats `json:"stats"` + HasContentParts bool // Flag to indicate which content type to marshal } -func (m RoleMsg) ToText(i int, cfg *config.Config) string { - icon := fmt.Sprintf("(%d)", i) - // check if already has role annotation (/completion makes them) - if !strings.HasPrefix(m.Content, m.Role+":") { - icon = fmt.Sprintf("(%d) <%s>: ", i, m.Role) +// MarshalJSON implements custom JSON marshaling for RoleMsg +// +//nolint:gocritic +func (m RoleMsg) MarshalJSON() ([]byte, error) { + if m.HasContentParts { + // Use structured content format + aux := struct { + Role string `json:"role"` + Content []any `json:"content"` + ToolCallID string `json:"tool_call_id,omitempty"` + ToolCall *ToolCall `json:"tool_call,omitempty"` + IsShellCommand bool `json:"is_shell_command,omitempty"` + KnownTo []string `json:"known_to,omitempty"` + Stats *ResponseStats `json:"stats,omitempty"` + }{ + Role: m.Role, + Content: m.ContentParts, + ToolCallID: m.ToolCallID, + ToolCall: m.ToolCall, + IsShellCommand: m.IsShellCommand, + KnownTo: m.KnownTo, + Stats: m.Stats, + } + return json.Marshal(aux) + } else { + // Use simple content format + aux := struct { + Role string `json:"role"` + Content string `json:"content"` + ToolCallID string `json:"tool_call_id,omitempty"` + ToolCall *ToolCall `json:"tool_call,omitempty"` + IsShellCommand bool `json:"is_shell_command,omitempty"` + KnownTo []string `json:"known_to,omitempty"` + Stats *ResponseStats `json:"stats,omitempty"` + }{ + Role: m.Role, + Content: m.Content, + ToolCallID: m.ToolCallID, + ToolCall: m.ToolCall, + IsShellCommand: m.IsShellCommand, + KnownTo: m.KnownTo, + Stats: m.Stats, + } + return json.Marshal(aux) } - textMsg := fmt.Sprintf("[-:-:b]%s[-:-:-]\n%s\n", icon, m.Content) - return strings.ReplaceAll(textMsg, "\n\n", "\n") } -func (m RoleMsg) ToPrompt() string { - return strings.ReplaceAll(fmt.Sprintf("%s:\n%s", m.Role, m.Content), "\n\n", "\n") +// UnmarshalJSON implements custom JSON unmarshaling for RoleMsg +func (m *RoleMsg) UnmarshalJSON(data []byte) error { + // First, try to unmarshal as structured content format + var structured struct { + Role string `json:"role"` + Content []any `json:"content"` + ToolCallID string `json:"tool_call_id,omitempty"` + ToolCall *ToolCall `json:"tool_call,omitempty"` + IsShellCommand bool `json:"is_shell_command,omitempty"` + KnownTo []string `json:"known_to,omitempty"` + Stats *ResponseStats `json:"stats,omitempty"` + } + if err := json.Unmarshal(data, &structured); err == nil && len(structured.Content) > 0 { + m.Role = structured.Role + m.ContentParts = structured.Content + m.ToolCallID = structured.ToolCallID + m.ToolCall = structured.ToolCall + m.IsShellCommand = structured.IsShellCommand + m.KnownTo = structured.KnownTo + m.Stats = structured.Stats + m.HasContentParts = true + return nil + } + + // Otherwise, unmarshal as simple content format + var simple struct { + Role string `json:"role"` + Content string `json:"content"` + ToolCallID string `json:"tool_call_id,omitempty"` + ToolCall *ToolCall `json:"tool_call,omitempty"` + IsShellCommand bool `json:"is_shell_command,omitempty"` + KnownTo []string `json:"known_to,omitempty"` + Stats *ResponseStats `json:"stats,omitempty"` + } + if err := json.Unmarshal(data, &simple); err != nil { + return err + } + m.Role = simple.Role + m.Content = simple.Content + m.ToolCallID = simple.ToolCallID + m.ToolCall = simple.ToolCall + m.IsShellCommand = simple.IsShellCommand + m.KnownTo = simple.KnownTo + m.Stats = simple.Stats + m.HasContentParts = false + return nil +} + +func (m *RoleMsg) ToPrompt() string { + var contentStr string + if !m.HasContentParts { + contentStr = m.Content + } else { + // For structured content, just take the text parts + var textParts []string + for _, part := range m.ContentParts { + switch p := part.(type) { + case TextContentPart: + if p.Type == "text" { + textParts = append(textParts, p.Text) + } + case ImageContentPart: + // skip images for text display + case map[string]any: + if partType, exists := p["type"]; exists && partType == "text" { + if textVal, textExists := p["text"]; textExists { + if textStr, isStr := textVal.(string); isStr { + textParts = append(textParts, textStr) + } + } + } + } + } + contentStr = strings.Join(textParts, " ") + " " + } + return strings.ReplaceAll(fmt.Sprintf("%s:\n%s", m.Role, contentStr), "\n\n", "\n") +} + +// NewRoleMsg creates a simple RoleMsg with string content +func NewRoleMsg(role, content string) RoleMsg { + return RoleMsg{ + Role: role, + Content: content, + HasContentParts: false, + } +} + +// NewMultimodalMsg creates a RoleMsg with structured content parts (text and images) +func NewMultimodalMsg(role string, contentParts []any) RoleMsg { + return RoleMsg{ + Role: role, + ContentParts: contentParts, + HasContentParts: true, + } +} + +// HasContent returns true if the message has either string content or structured content parts +func (m *RoleMsg) HasContent() bool { + if m.Content != "" { + return true + } + if m.HasContentParts && len(m.ContentParts) > 0 { + return true + } + return false +} + +// IsContentParts returns true if the message uses structured content parts +func (m *RoleMsg) IsContentParts() bool { + return m.HasContentParts +} + +// GetContentParts returns the content parts of the message +func (m *RoleMsg) GetContentParts() []any { + return m.ContentParts +} + +// Copy creates a copy of the RoleMsg with all fields +func (m *RoleMsg) Copy() RoleMsg { + return RoleMsg{ + Role: m.Role, + Content: m.Content, + ContentParts: m.ContentParts, + ToolCallID: m.ToolCallID, + KnownTo: m.KnownTo, + Stats: m.Stats, + HasContentParts: m.HasContentParts, + ToolCall: m.ToolCall, + IsShellCommand: m.IsShellCommand, + } +} + +// GetText returns the text content of the message, handling both +// simple Content and multimodal ContentParts formats. +func (m *RoleMsg) GetText() string { + if !m.HasContentParts { + return m.Content + } + var textParts []string + for _, part := range m.ContentParts { + switch p := part.(type) { + case TextContentPart: + if p.Type == "text" { + textParts = append(textParts, p.Text) + } + case map[string]any: + if partType, exists := p["type"]; exists { + if partType == "text" { + if textVal, textExists := p["text"]; textExists { + if textStr, isStr := textVal.(string); isStr { + textParts = append(textParts, textStr) + } + } + } + } + } + } + return strings.Join(textParts, " ") +} + +// SetText updates the text content of the message. If the message has +// ContentParts (multimodal), it updates the text parts while preserving +// images. If not, it sets the simple Content field. +func (m *RoleMsg) SetText(text string) { + if !m.HasContentParts { + m.Content = text + return + } + var newParts []any + for _, part := range m.ContentParts { + switch p := part.(type) { + case TextContentPart: + if p.Type == "text" { + p.Text = text + newParts = append(newParts, p) + } else { + newParts = append(newParts, p) + } + case map[string]any: + if partType, exists := p["type"]; exists && partType == "text" { + p["text"] = text + newParts = append(newParts, p) + } else { + newParts = append(newParts, p) + } + default: + newParts = append(newParts, part) + } + } + m.ContentParts = newParts +} + +// AddTextPart adds a text content part to the message +func (m *RoleMsg) AddTextPart(text string) { + if !m.HasContentParts { + // Convert to content parts format + if m.Content != "" { + m.ContentParts = []any{TextContentPart{Type: "text", Text: m.Content}} + } else { + m.ContentParts = []any{} + } + m.HasContentParts = true + } + textPart := TextContentPart{Type: "text", Text: text} + m.ContentParts = append(m.ContentParts, textPart) +} + +// AddImagePart adds an image content part to the message +func (m *RoleMsg) AddImagePart(imageURL, imagePath string) { + if !m.HasContentParts { + // Convert to content parts format + if m.Content != "" { + m.ContentParts = []any{TextContentPart{Type: "text", Text: m.Content}} + } else { + m.ContentParts = []any{} + } + m.HasContentParts = true + } + imagePart := ImageContentPart{ + Type: "image_url", + Path: imagePath, // Store the original file path + ImageURL: struct { + URL string `json:"url"` + }{URL: imageURL}, + } + m.ContentParts = append(m.ContentParts, imagePart) +} + +// CreateImageURLFromPath creates a data URL from an image file path +func CreateImageURLFromPath(imagePath string) (string, error) { + // Read the image file + data, err := os.ReadFile(imagePath) + if err != nil { + return "", err + } + // Determine the image format based on file extension + var mimeType string + switch { + case strings.HasSuffix(strings.ToLower(imagePath), ".png"): + mimeType = "image/png" + case strings.HasSuffix(strings.ToLower(imagePath), ".jpg"): + fallthrough + case strings.HasSuffix(strings.ToLower(imagePath), ".jpeg"): + mimeType = "image/jpeg" + case strings.HasSuffix(strings.ToLower(imagePath), ".gif"): + mimeType = "image/gif" + case strings.HasSuffix(strings.ToLower(imagePath), ".webp"): + mimeType = "image/webp" + default: + mimeType = "image/jpeg" // default + } + // Encode to base64 + encoded := base64.StdEncoding.EncodeToString(data) + // Create data URL + return fmt.Sprintf("data:%s;base64,%s", mimeType, encoded), nil } type ChatBody struct { @@ -76,31 +419,51 @@ type ChatBody struct { Messages []RoleMsg `json:"messages"` } -type ChatToolsBody struct { - Model string `json:"model"` - Messages []RoleMsg `json:"messages"` - Tools []struct { - Type string `json:"type"` - Function struct { - Name string `json:"name"` - Description string `json:"description"` - Parameters struct { - Type string `json:"type"` - Properties struct { - Location struct { - Type string `json:"type"` - Description string `json:"description"` - } `json:"location"` - Unit struct { - Type string `json:"type"` - Enum []string `json:"enum"` - } `json:"unit"` - } `json:"properties"` - Required []string `json:"required"` - } `json:"parameters"` - } `json:"function"` - } `json:"tools"` - ToolChoice string `json:"tool_choice"` +func (cb *ChatBody) Rename(oldname, newname string) { + for i := range cb.Messages { + cb.Messages[i].Content = strings.ReplaceAll(cb.Messages[i].Content, oldname, newname) + cb.Messages[i].Role = strings.ReplaceAll(cb.Messages[i].Role, oldname, newname) + } +} + +func (cb *ChatBody) ListRoles() []string { + namesMap := make(map[string]struct{}) + for i := range cb.Messages { + namesMap[cb.Messages[i].Role] = struct{}{} + } + resp := make([]string, len(namesMap)) + i := 0 + for k := range namesMap { + resp[i] = k + i++ + } + return resp +} + +func (cb *ChatBody) MakeStopSlice() []string { + return cb.MakeStopSliceExcluding("", cb.ListRoles()) +} + +func (cb *ChatBody) MakeStopSliceExcluding( + excludeRole string, roleList []string, +) []string { + ss := []string{} + for _, role := range roleList { + // Skip the excluded role (typically the current speaker) + if role == excludeRole { + continue + } + // Add multiple variations to catch different formatting + ss = append(ss, + role+":\n", // Most common: role with newline + role+":", // Role with colon but no newline + role+": ", // Role with colon and single space + role+": ", // Role with colon and double space (common tokenization) + role+": \n", // Role with colon and double space (common tokenization) + role+": ", // Role with colon and triple space + ) + } + return ss } type EmbeddingResp struct { @@ -122,28 +485,43 @@ type EmbeddingResp struct { // } `json:"data"` // } -type LLMModels struct { - Object string `json:"object"` - Data []struct { - ID string `json:"id"` - Object string `json:"object"` - Created int `json:"created"` - OwnedBy string `json:"owned_by"` - Meta struct { - VocabType int `json:"vocab_type"` - NVocab int `json:"n_vocab"` - NCtxTrain int `json:"n_ctx_train"` - NEmbd int `json:"n_embd"` - NParams int64 `json:"n_params"` - Size int64 `json:"size"` - } `json:"meta"` - } `json:"data"` +// === tools models + +type ToolArgProps struct { + Type string `json:"type"` + Description string `json:"description"` +} + +type ToolFuncParams struct { + Type string `json:"type"` + Properties map[string]ToolArgProps `json:"properties"` + Required []string `json:"required"` +} + +type ToolFunc struct { + Name string `json:"name"` + Description string `json:"description"` + Parameters ToolFuncParams `json:"parameters"` +} + +type Tool struct { + Type string `json:"type"` + Function ToolFunc `json:"function"` +} + +type OpenAIReq struct { + *ChatBody + Tools []Tool `json:"tools"` } +// === + type LlamaCPPReq struct { - Stream bool `json:"stream"` - // Messages []RoleMsg `json:"messages"` - Prompt string `json:"prompt"` + Model string `json:"model"` + Stream bool `json:"stream"` + // For multimodal requests, prompt should be an object with prompt_string and multimodal_data + // For regular requests, prompt is a string + Prompt any `json:"prompt"` // Can be string or object with prompt_string and multimodal_data Temperature float32 `json:"temperature"` DryMultiplier float32 `json:"dry_multiplier"` Stop []string `json:"stop"` @@ -168,21 +546,36 @@ type LlamaCPPReq struct { // Samplers string `json:"samplers"` } -func NewLCPReq(prompt string, cfg *config.Config, props map[string]float32) LlamaCPPReq { +type PromptObject struct { + PromptString string `json:"prompt_string"` + MultimodalData []string `json:"multimodal_data,omitempty"` + // Alternative field name used by some llama.cpp implementations + ImageData []string `json:"image_data,omitempty"` // For compatibility +} + +func NewLCPReq(prompt, model string, multimodalData []string, props map[string]float32, stopStrings []string) LlamaCPPReq { + var finalPrompt any + if len(multimodalData) > 0 { + // When multimodal data is present, use the object format as per Python example: + // { "prompt": { "prompt_string": "...", "multimodal_data": [...] } } + finalPrompt = PromptObject{ + PromptString: prompt, + MultimodalData: multimodalData, + ImageData: multimodalData, // Also populate for compatibility with different llama.cpp versions + } + } else { + // When no multimodal data, use plain string + finalPrompt = prompt + } return LlamaCPPReq{ - Stream: true, - Prompt: prompt, - // Temperature: 0.8, - // DryMultiplier: 0.5, + Model: model, + Stream: true, + Prompt: finalPrompt, Temperature: props["temperature"], DryMultiplier: props["dry_multiplier"], + Stop: stopStrings, MinP: props["min_p"], NPredict: int32(props["n_predict"]), - Stop: []string{ - cfg.UserRole + ":\n", "<|im_end|>", - cfg.ToolRole + ":\n", - cfg.AssistantRole + ":\n", - }, } } @@ -190,3 +583,59 @@ type LlamaCPPResp struct { Content string `json:"content"` Stop bool `json:"stop"` } + +type LCPModels struct { + Data []struct { + ID string `json:"id"` + Object string `json:"object"` + OwnedBy string `json:"owned_by"` + Created int `json:"created"` + InCache bool `json:"in_cache"` + Path string `json:"path"` + Status struct { + Value string `json:"value"` + Args []string `json:"args"` + } `json:"status"` + } `json:"data"` + Object string `json:"object"` +} + +func (lcp *LCPModels) ListModels() []string { + resp := make([]string, 0, len(lcp.Data)) + for _, model := range lcp.Data { + resp = append(resp, model.ID) + } + return resp +} + +func (lcp *LCPModels) HasVision(modelID string) bool { + for _, m := range lcp.Data { + if m.ID == modelID { + args := m.Status.Args + for i := 0; i < len(args)-1; i++ { + if args[i] == "--mmproj" { + return true + } + } + } + } + return false +} + +type ResponseStats struct { + Tokens int + Duration float64 + TokensPerSec float64 +} + +type ChatRoundReq struct { + UserMsg string + Role string + Regen bool + Resume bool +} + +type MultimodalToolResp struct { + Type string `json:"type"` + Parts []map[string]string `json:"parts"` +} diff --git a/models/openrouter.go b/models/openrouter.go new file mode 100644 index 0000000..2dd49cc --- /dev/null +++ b/models/openrouter.go @@ -0,0 +1,187 @@ +package models + +// openrouter +// https://openrouter.ai/docs/api-reference/completion +type OpenRouterCompletionReq struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + Stream bool `json:"stream"` + Temperature float32 `json:"temperature"` + Stop []string `json:"stop"` // not present in docs + MinP float32 `json:"min_p"` + NPredict int32 `json:"max_tokens"` +} + +func NewOpenRouterCompletionReq(model, prompt string, props map[string]float32, stopStrings []string) OpenRouterCompletionReq { + return OpenRouterCompletionReq{ + Stream: true, + Prompt: prompt, + Temperature: props["temperature"], + MinP: props["min_p"], + NPredict: int32(props["n_predict"]), + Stop: stopStrings, + Model: model, + } +} + +type OpenRouterChatReq struct { + Messages []RoleMsg `json:"messages"` + Model string `json:"model"` + Stream bool `json:"stream"` + Temperature float32 `json:"temperature"` + MinP float32 `json:"min_p"` + NPredict int32 `json:"max_tokens"` + Tools []Tool `json:"tools"` + Reasoning *ReasoningConfig `json:"reasoning,omitempty"` +} + +type ReasoningConfig struct { + Effort string `json:"effort,omitempty"` // xhigh, high, medium, low, minimal, none + Summary string `json:"summary,omitempty"` // auto, concise, detailed +} + +func NewOpenRouterChatReq(cb ChatBody, props map[string]float32, reasoningEffort string) OpenRouterChatReq { + req := OpenRouterChatReq{ + Messages: cb.Messages, + Model: cb.Model, + Stream: cb.Stream, + Temperature: props["temperature"], + MinP: props["min_p"], + NPredict: int32(props["n_predict"]), + } + // Only include reasoning config if effort is specified and not "none" + if reasoningEffort != "" && reasoningEffort != "none" { + req.Reasoning = &ReasoningConfig{ + Effort: reasoningEffort, + } + } + return req +} + +type OpenRouterChatRespNonStream struct { + ID string `json:"id"` + Provider string `json:"provider"` + Model string `json:"model"` + Object string `json:"object"` + Created int `json:"created"` + Choices []struct { + Logprobs any `json:"logprobs"` + FinishReason string `json:"finish_reason"` + NativeFinishReason string `json:"native_finish_reason"` + Index int `json:"index"` + Message struct { + Role string `json:"role"` + Content string `json:"content"` + Refusal any `json:"refusal"` + Reasoning any `json:"reasoning"` + ToolCalls []ToolDeltaResp `json:"tool_calls"` + } `json:"message"` + } `json:"choices"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage"` +} + +type OpenRouterChatResp struct { + ID string `json:"id"` + Provider string `json:"provider"` + Model string `json:"model"` + Object string `json:"object"` + Created int `json:"created"` + Choices []struct { + Index int `json:"index"` + Delta struct { + Role string `json:"role"` + Content string `json:"content"` + Reasoning string `json:"reasoning"` + ToolCalls []ToolDeltaResp `json:"tool_calls"` + } `json:"delta"` + FinishReason string `json:"finish_reason"` + NativeFinishReason string `json:"native_finish_reason"` + Logprobs any `json:"logprobs"` + } `json:"choices"` +} + +type OpenRouterCompletionResp struct { + ID string `json:"id"` + Provider string `json:"provider"` + Model string `json:"model"` + Object string `json:"object"` + Created int `json:"created"` + Choices []struct { + Text string `json:"text"` + FinishReason string `json:"finish_reason"` + NativeFinishReason string `json:"native_finish_reason"` + Logprobs any `json:"logprobs"` + } `json:"choices"` +} + +type ORModel struct { + ID string `json:"id"` + CanonicalSlug string `json:"canonical_slug"` + HuggingFaceID string `json:"hugging_face_id"` + Name string `json:"name"` + Created int `json:"created"` + Description string `json:"description"` + ContextLength int `json:"context_length"` + Architecture struct { + Modality string `json:"modality"` + InputModalities []string `json:"input_modalities"` + OutputModalities []string `json:"output_modalities"` + Tokenizer string `json:"tokenizer"` + InstructType any `json:"instruct_type"` + } `json:"architecture"` + Pricing struct { + Prompt string `json:"prompt"` + Completion string `json:"completion"` + Request string `json:"request"` + Image string `json:"image"` + Audio string `json:"audio"` + WebSearch string `json:"web_search"` + InternalReasoning string `json:"internal_reasoning"` + } `json:"pricing,omitempty"` + TopProvider struct { + ContextLength int `json:"context_length"` + MaxCompletionTokens int `json:"max_completion_tokens"` + IsModerated bool `json:"is_moderated"` + } `json:"top_provider"` + PerRequestLimits any `json:"per_request_limits"` + SupportedParameters []string `json:"supported_parameters"` +} + +type ORModels struct { + Data []ORModel `json:"data"` +} + +func (orm *ORModels) ListModels(free bool) []string { + resp := []string{} + for i := range orm.Data { + model := &orm.Data[i] // Take address of element to avoid copying + if free { + if model.Pricing.Prompt == "0" && model.Pricing.Completion == "0" { + // treat missing request as free + if model.Pricing.Request == "" || model.Pricing.Request == "0" { + resp = append(resp, model.ID) + } + } + } else { + resp = append(resp, model.ID) + } + } + return resp +} + +func (orm *ORModels) HasVision(modelID string) bool { + for i := range orm.Data { + if orm.Data[i].ID == modelID { + for _, mod := range orm.Data[i].Architecture.InputModalities { + if mod == "image" { + return true + } + } + } + } + return false +} diff --git a/models/openrouter_test.go b/models/openrouter_test.go new file mode 100644 index 0000000..63990b6 --- /dev/null +++ b/models/openrouter_test.go @@ -0,0 +1,96 @@ +package models + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +func TestORModelsListModels(t *testing.T) { + t.Run("unit test with hardcoded data", func(t *testing.T) { + jsonData := `{ + "data": [ + { + "id": "model/free", + "pricing": { + "prompt": "0", + "completion": "0" + } + }, + { + "id": "model/paid", + "pricing": { + "prompt": "0.001", + "completion": "0.002" + } + }, + { + "id": "model/request-zero", + "pricing": { + "prompt": "0", + "completion": "0", + "request": "0" + } + }, + { + "id": "model/request-nonzero", + "pricing": { + "prompt": "0", + "completion": "0", + "request": "0.5" + } + } + ] + }` + var models ORModels + if err := json.Unmarshal([]byte(jsonData), &models); err != nil { + t.Fatalf("failed to unmarshal test data: %v", err) + } + freeModels := models.ListModels(true) + if len(freeModels) != 2 { + t.Errorf("expected 2 free models, got %d: %v", len(freeModels), freeModels) + } + expectedFree := map[string]bool{"model/free": true, "model/request-zero": true} + for _, id := range freeModels { + if !expectedFree[id] { + t.Errorf("unexpected free model ID: %s", id) + } + } + allModels := models.ListModels(false) + if len(allModels) != 4 { + t.Errorf("expected 4 total models, got %d", len(allModels)) + } + }) + t.Run("integration with or_models.json", func(t *testing.T) { + // Attempt to load the real data file from the project root + path := filepath.Join("..", "or_models.json") + data, err := os.ReadFile(path) + if err != nil { + t.Skip("or_models.json not found, skipping integration test") + } + var models ORModels + if err := json.Unmarshal(data, &models); err != nil { + t.Fatalf("failed to unmarshal %s: %v", path, err) + } + freeModels := models.ListModels(true) + if len(freeModels) == 0 { + t.Error("expected at least one free model, got none") + } + allModels := models.ListModels(false) + if len(allModels) == 0 { + t.Error("expected at least one model") + } + // Ensure free models are subset of all models + freeSet := make(map[string]bool) + for _, id := range freeModels { + freeSet[id] = true + } + for _, id := range freeModels { + if !freeSet[id] { + t.Errorf("free model %s not found in all models", id) + } + } + t.Logf("found %d free models out of %d total models", len(freeModels), len(allModels)) + }) +}
\ No newline at end of file |