Enha: or img attachment

1 files changed, 147 insertions, 22 deletions

diff --git a/llm.go b/llm.go
index d22ede2..217a87f 100644
--- a/llm.go
+++ b/llm.go
@@ -3,8 +3,10 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"gf-lt/models"
 	"io"
+	"os"
 	"strings"
 )
 
@@ -76,6 +78,13 @@ type OpenRouterChat struct {
 	Model string
 }
 
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
 func (lcp LlamaCPPeer) GetToken() string {
 	return ""
 }
@@ -180,24 +189,29 @@ func (op OpenAIer) ParseChunk(data []byte) (*models.TextChunk, error) {
 
 func (op OpenAIer) FormMsg(msg, role string, resume bool) (io.Reader, error) {
 	logger.Debug("formmsg openaier", "link", cfg.CurrentAPI)
+
+	// Capture the image attachment path at the beginning to avoid race conditions
+	// with API rotation that might clear the global variable
+	localImageAttachmentPath := imageAttachmentPath
+
 	if msg != "" { // otherwise let the bot continue
 		// Create the message with support for multimodal content
 		var newMsg models.RoleMsg
 		// Check if we have an image to add to this message
-		if imageAttachmentPath != "" {
+		if localImageAttachmentPath != "" {
 			// Create a multimodal message with both text and image
 			newMsg = models.NewMultimodalMsg(role, []interface{}{})
 			// Add the text content
 			newMsg.AddTextPart(msg)
 			// Add the image content
-			imageURL, err := models.CreateImageURLFromPath(imageAttachmentPath)
+			imageURL, err := models.CreateImageURLFromPath(localImageAttachmentPath)
 			if err != nil {
-				logger.Error("failed to create image URL from path", "error", err, "path", imageAttachmentPath)
+				logger.Error("failed to create image URL from path", "error", err, "path", localImageAttachmentPath)
 				// If image processing fails, fall back to simple text message
 				newMsg = models.NewRoleMsg(role, msg)
-				imageAttachmentPath = "" // Clear the attachment
 			} else {
 				newMsg.AddImagePart(imageURL)
+				// Only clear the global image attachment after successfully processing it in this API call
 				imageAttachmentPath = "" // Clear the attachment after use
 			}
 		} else {
@@ -478,6 +492,11 @@ func (or OpenRouterChat) GetToken() string {
 
 func (or OpenRouterChat) FormMsg(msg, role string, resume bool) (io.Reader, error) {
 	logger.Debug("formmsg open router completion", "link", cfg.CurrentAPI)
+
+	// Capture the image attachment path at the beginning to avoid race conditions
+	// with API rotation that might clear the global variable
+	localImageAttachmentPath := imageAttachmentPath
+
 	if cfg.ToolUse && !resume {
 		// prompt += "\n" + cfg.ToolRole + ":\n" + toolSysMsg
 		// add to chat body
@@ -486,21 +505,36 @@ func (or OpenRouterChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
 	if msg != "" { // otherwise let the bot continue
 		var newMsg models.RoleMsg
 		// Check if we have an image to add to this message
-		if imageAttachmentPath != "" {
-			// Create a multimodal message with both text and image
-			newMsg = models.NewMultimodalMsg(role, []interface{}{})
-			// Add the text content
-			newMsg.AddTextPart(msg)
-			// Add the image content
-			imageURL, err := models.CreateImageURLFromPath(imageAttachmentPath)
-			if err != nil {
-				logger.Error("failed to create image URL from path", "error", err, "path", imageAttachmentPath)
-				// If image processing fails, fall back to simple text message
+		logger.Debug("checking for image attachment", "imageAttachmentPath", localImageAttachmentPath, "msg", msg, "role", role)
+		if localImageAttachmentPath != "" {
+			logger.Info("processing image attachment for OpenRouter", "path", localImageAttachmentPath, "msg", msg)
+			// Check if file exists before attempting to create image URL
+			if _, err := os.Stat(localImageAttachmentPath); os.IsNotExist(err) {
+				logger.Error("image file does not exist", "path", localImageAttachmentPath)
+				// Fallback to simple text message
+				newMsg = models.NewRoleMsg(role, msg)
+			} else if err != nil {
+				logger.Error("error checking image file", "path", localImageAttachmentPath, "error", err)
+				// Fallback to simple text message
 				newMsg = models.NewRoleMsg(role, msg)
-				imageAttachmentPath = "" // Clear the attachment
 			} else {
-				newMsg.AddImagePart(imageURL)
-				imageAttachmentPath = "" // Clear the attachment after use
+				logger.Debug("image file exists, proceeding to create URL", "path", localImageAttachmentPath)
+				// Create a multimodal message with both text and image
+				newMsg = models.NewMultimodalMsg(role, []interface{}{})
+				// Add the text content
+				newMsg.AddTextPart(msg)
+				// Add the image content
+				imageURL, err := models.CreateImageURLFromPath(localImageAttachmentPath)
+				if err != nil {
+					logger.Error("failed to create image URL from path", "error", err, "path", localImageAttachmentPath)
+					// If image processing fails, fall back to simple text message
+					newMsg = models.NewRoleMsg(role, msg)
+				} else {
+					logger.Info("image URL created successfully for OpenRouter", "imageURL", imageURL[:min(len(imageURL), 50)]+"...")
+					newMsg.AddImagePart(imageURL)
+					// Only clear the global image attachment after successfully processing it in this API call
+					imageAttachmentPath = "" // Clear the attachment after use
+				}
 			}
 		} else {
 			// Create a simple text message
@@ -520,27 +554,118 @@ func (or OpenRouterChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
 		}
 	}
 	// Create copy of chat body with standardized user role
-	// modifiedBody := *chatBody
 	bodyCopy := &models.ChatBody{
 		Messages: make([]models.RoleMsg, len(chatBody.Messages)),
 		Model:    chatBody.Model,
 		Stream:   chatBody.Stream,
 	}
-	// modifiedBody.Messages = make([]models.RoleMsg, len(chatBody.Messages))
+
 	for i, msg := range chatBody.Messages {
 		logger.Debug("checking roles", "#", i, "role", msg.Role)
-		if msg.Role == cfg.UserRole || i == 1 {
-			bodyCopy.Messages[i].Role = "user"
-			logger.Debug("replaced role in body", "#", i)
+		// Check if this message has content parts (multimodal) by attempting to marshal and checking structure
+		msgBytes, err := json.Marshal(msg)
+		if err != nil {
+			logger.Error("failed to serialize message for inspection", "error", err)
+			// Fallback to direct assignment
+			bodyCopy.Messages[i] = msg
 		} else {
+			// Try to deserialize to check if it has content parts
+			var tempMsg map[string]interface{}
+			if err := json.Unmarshal(msgBytes, &tempMsg); err != nil {
+				logger.Error("failed to inspect message structure", "error", err)
+				bodyCopy.Messages[i] = msg
+			} else {
+				// Check if content is an array (indicating content parts) or string (simple content)
+				if content, ok := tempMsg["content"]; ok {
+					if _, isArray := content.([]interface{}); isArray {
+						logger.Info("multimodal message detected", "#", i, "role", msg.Role)
+						// Deserialize to RoleMsg to access ContentParts
+						var detailedMsg models.RoleMsg
+						if err := json.Unmarshal(msgBytes, &detailedMsg); err == nil {
+							if len(detailedMsg.ContentParts) > 0 {
+								for j, part := range detailedMsg.ContentParts {
+									if textPart, ok := part.(models.TextContentPart); ok {
+										logger.Debug("text content part", "msg#", i, "part#", j, "text", textPart.Text)
+									} else if imgPart, ok := part.(models.ImageContentPart); ok {
+										logger.Info("image content part", "msg#", i, "part#", j, "url", imgPart.ImageURL.URL[:min(len(imgPart.ImageURL.URL), 50)]+"...")
+									} else {
+										logger.Debug("other content part", "msg#", i, "part#", j, "type", fmt.Sprintf("%T", part))
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+
+		// Create a proper copy of the message that preserves all internal state
+		// First, serialize and deserialize to ensure content parts are preserved
+		copyMsgBytes, err := json.Marshal(msg)
+		if err != nil {
+			logger.Error("failed to serialize message", "error", err)
+			// Fallback to direct assignment
 			bodyCopy.Messages[i] = msg
+		} else {
+			// Deserialize back to preserve all internal state
+			var copiedMsg models.RoleMsg
+			err := json.Unmarshal(copyMsgBytes, &copiedMsg)
+			if err != nil {
+				logger.Error("failed to deserialize message", "error", err)
+				// Fallback to direct assignment
+				bodyCopy.Messages[i] = msg
+			} else {
+				bodyCopy.Messages[i] = copiedMsg
+			}
+		}
+
+		// Standardize role if it's a user role or first message
+		if bodyCopy.Messages[i].Role == cfg.UserRole || i == 1 {
+			bodyCopy.Messages[i].Role = "user"
+			logger.Debug("replaced role in body", "#", i)
 		}
 	}
+
+	// Log the final request body before sending to OpenRouter
 	orBody := models.NewOpenRouterChatReq(*bodyCopy, defaultLCPProps)
 	data, err := json.Marshal(orBody)
 	if err != nil {
 		logger.Error("failed to form a msg", "error", err)
 		return nil, err
 	}
+
+	logger.Info("OpenRouter request prepared", "messages_count", len(orBody.Messages))
+	for i, msg := range orBody.Messages {
+		// Check if this final message has content parts (multimodal)
+		msgBytes, err := json.Marshal(msg)
+		if err == nil {
+			var tempMsg map[string]interface{}
+			if err := json.Unmarshal(msgBytes, &tempMsg); err == nil {
+				if content, ok := tempMsg["content"]; ok {
+					if _, isArray := content.([]interface{}); isArray {
+						logger.Debug("final message", "#", i, "role", msg.Role, "hasContentParts", true)
+						// Deserialize to access content parts
+						var detailedMsg models.RoleMsg
+						if err := json.Unmarshal(msgBytes, &detailedMsg); err == nil {
+							if len(detailedMsg.ContentParts) > 0 {
+								for j, part := range detailedMsg.ContentParts {
+									if textPart, ok := part.(models.TextContentPart); ok {
+										logger.Debug("final text part", "msg#", i, "part#", j, "text", textPart.Text)
+									} else if imgPart, ok := part.(models.ImageContentPart); ok {
+										logger.Info("final image part sent to OpenRouter", "msg#", i, "part#", j, "url", imgPart.ImageURL.URL[:min(len(imgPart.ImageURL.URL), 50)]+"...")
+									} else {
+										logger.Debug("final other part", "msg#", i, "part#", j, "type", fmt.Sprintf("%T", part))
+									}
+								}
+							}
+						}
+					} else {
+						logger.Debug("final message", "#", i, "role", msg.Role, "hasContentParts", false)
+					}
+				}
+			}
+		}
+	}
+
 	return bytes.NewReader(data), nil
 }