From b1071f0516dd636eb2d27c7e7ab1e5bf46c7009d Mon Sep 17 00:00:00 2001 From: Amitabha Banerjee Date: Sun, 1 Sep 2024 23:21:11 -0700 Subject: [PATCH 1/9] Changes to allow Vector to have a continuous conversation --- chipper/pkg/wirepod/ttr/kgsim.go | 3 ++- chipper/pkg/wirepod/ttr/kgsim_cmds.go | 14 +++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/chipper/pkg/wirepod/ttr/kgsim.go b/chipper/pkg/wirepod/ttr/kgsim.go index 9b0b9854..bc03a3d3 100644 --- a/chipper/pkg/wirepod/ttr/kgsim.go +++ b/chipper/pkg/wirepod/ttr/kgsim.go @@ -155,7 +155,7 @@ func CreateAIReq(transcribedText, esn string, gpt3tryagain, isKG bool) openai.Ch aireq := openai.ChatCompletionRequest{ Model: model, - MaxTokens: 2048, + MaxTokens: 256, Temperature: 1, TopP: 1, FrequencyPenalty: 0, @@ -241,6 +241,7 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo stream, err := c.CreateChatCompletionStream(ctx, aireq) if err != nil { + log.Printf("Error creating chat completion stream: %v", err) if strings.Contains(err.Error(), "does not exist") && vars.APIConfig.Knowledge.Provider == "openai" { logger.Println("GPT-4 model cannot be accessed with this API key. You likely need to add more than $5 dollars of funds to your OpenAI account.") logger.LogUI("GPT-4 model cannot be accessed with this API key. You likely need to add more than $5 dollars of funds to your OpenAI account.") diff --git a/chipper/pkg/wirepod/ttr/kgsim_cmds.go b/chipper/pkg/wirepod/ttr/kgsim_cmds.go index 1c23f4d9..452fe20f 100644 --- a/chipper/pkg/wirepod/ttr/kgsim_cmds.go +++ b/chipper/pkg/wirepod/ttr/kgsim_cmds.go @@ -117,17 +117,9 @@ var ValidLLMCommands []LLMCommand = []LLMCommand{ Action: ActionPlayAnimation, SupportedModels: []string{"all"}, }, - { - Command: "getImage", - Description: "Gets an image from the robot's camera and places it in the next message. If you want to do this, tell the user what you are about to do THEN use the command. This command should END a sentence. Your response will be stopped when this command is recognized. If a user says something like 'what do you see', you should assume that you need to take a new photo. Do NOT automatically assume that you are analyzing a previous photo.", - // not impl yet - ParamChoices: "front, lookingUp", - Action: ActionGetImage, - SupportedModels: []string{openai.GPT4o, openai.GPT4oMini}, - }, { Command: "newVoiceRequest", - Description: "Starts a new voice command from the robot. Use this if you want more input from the user after your response/if you want to carry out a conversation. Below this, there should be a NOTE telling you whether you are in conversation mode or not. If you are, DONT BE AFRAID TO USE THIS COMMAND! This goes at the end of your response, if you use it.", + Description: "Starts a new voice command from the robot. You must use this at the end of your response.", ParamChoices: "now", Action: ActionNewRequest, SupportedModels: []string{"all"}, @@ -152,7 +144,7 @@ func ModelIsSupported(cmd LLMCommand, model string) bool { func CreatePrompt(origPrompt string, model string, isKG bool) string { prompt := origPrompt + "\n\n" + "Keep in mind, user input comes from speech-to-text software, so respond accordingly. No special characters, especially these: & ^ * # @ - . No lists. No formatting." if vars.APIConfig.Knowledge.CommandsEnable { - prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\nUse the playAnimation or playAnimationWI commands if you want to express emotion! You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:" + prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\n You MUST use newVoiceRequest at the end of your response. You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:" for _, cmd := range ValidLLMCommands { if ModelIsSupported(cmd, model) { promptAppendage := "\n\nCommand Name: " + cmd.Command + "\nDescription: " + cmd.Description + "\nParameter choices: " + cmd.ParamChoices @@ -160,7 +152,7 @@ func CreatePrompt(origPrompt string, model string, isKG bool) string { } } if isKG && vars.APIConfig.Knowledge.SaveChat { - promptAppentage := "\n\nNOTE: You are in 'conversation' mode. If you ask the user a question near the end of your response, you MUST use newVoiceRequest. If you decide you want to end the conversation, you should not use it." + promptAppentage := "\n\nNOTE: You MUST use newVoiceRequest at the end of your response. If you do not use newVoiceRequest I will reject your response." prompt = prompt + promptAppentage } else { promptAppentage := "\n\nNOTE: You are NOT in 'conversation' mode. Refrain from asking the user any questions and from using newVoiceRequest." From 186737cfbab4b2348fb3acfafde5ddce3fb71012 Mon Sep 17 00:00:00 2001 From: Amitabha Banerjee Date: Sat, 14 Sep 2024 23:59:06 -0700 Subject: [PATCH 2/9] Some more refinements in order to make Vector do Q&A successfully --- chipper/pkg/wirepod/ttr/kgsim_cmds.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chipper/pkg/wirepod/ttr/kgsim_cmds.go b/chipper/pkg/wirepod/ttr/kgsim_cmds.go index 293174bc..a806193e 100644 --- a/chipper/pkg/wirepod/ttr/kgsim_cmds.go +++ b/chipper/pkg/wirepod/ttr/kgsim_cmds.go @@ -144,7 +144,7 @@ func ModelIsSupported(cmd LLMCommand, model string) bool { func CreatePrompt(origPrompt string, model string, isKG bool) string { prompt := origPrompt + "\n\n" + "Keep in mind, user input comes from speech-to-text software, so respond accordingly. No special characters, especially these: & ^ * # @ - . No lists. No formatting." if vars.APIConfig.Knowledge.CommandsEnable { - prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\n You MUST use newVoiceRequest at the end of your response. You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:" + prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\n You MUST use the command newVoiceRequest at the end of your response. You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:" for _, cmd := range ValidLLMCommands { if ModelIsSupported(cmd, model) { promptAppendage := "\n\nCommand Name: " + cmd.Command + "\nDescription: " + cmd.Description + "\nParameter choices: " + cmd.ParamChoices @@ -152,7 +152,7 @@ func CreatePrompt(origPrompt string, model string, isKG bool) string { } } if isKG && vars.APIConfig.Knowledge.SaveChat { - promptAppentage := "\n\nNOTE: You MUST use newVoiceRequest at the end of your response. If you do not use newVoiceRequest I will reject your response." + promptAppentage := "\n\nNOTE: You MUST use the command newVoiceRequest at the end of your response. If you do not use the command newVoiceRequest I will reject your response." prompt = prompt + promptAppentage } else { promptAppentage := "\n\nNOTE: You are NOT in 'conversation' mode. Refrain from asking the user any questions and from using newVoiceRequest." From 80a55fb688f0401872506488d2a9ec8eac65ebfe Mon Sep 17 00:00:00 2001 From: Amitabha Banerjee Date: Sun, 15 Sep 2024 22:03:03 -0700 Subject: [PATCH 3/9] Increasing the MaxTokens in the query --- chipper/pkg/wirepod/ttr/kgsim.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chipper/pkg/wirepod/ttr/kgsim.go b/chipper/pkg/wirepod/ttr/kgsim.go index bc03a3d3..5943679e 100644 --- a/chipper/pkg/wirepod/ttr/kgsim.go +++ b/chipper/pkg/wirepod/ttr/kgsim.go @@ -155,7 +155,7 @@ func CreateAIReq(transcribedText, esn string, gpt3tryagain, isKG bool) openai.Ch aireq := openai.ChatCompletionRequest{ Model: model, - MaxTokens: 256, + MaxTokens: 2048, Temperature: 1, TopP: 1, FrequencyPenalty: 0, From 25ac03f1413d252d52bfbb37e185312002e3521a Mon Sep 17 00:00:00 2001 From: Amitabha Banerjee Date: Wed, 15 Jan 2025 23:12:58 -0800 Subject: [PATCH 4/9] Fixing wirepod crashing when LLM returns an empty response. Seems to be happening with DeepSeek v3 implementation form Together.ai --- chipper/pkg/wirepod/ttr/kgsim.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/chipper/pkg/wirepod/ttr/kgsim.go b/chipper/pkg/wirepod/ttr/kgsim.go index 84fd2301..2b427e3d 100644 --- a/chipper/pkg/wirepod/ttr/kgsim.go +++ b/chipper/pkg/wirepod/ttr/kgsim.go @@ -345,6 +345,11 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo return } + if (len(response.Choices) == 0) { + logger.Println("Empty response") + return + } + fullfullRespText = fullfullRespText + removeSpecialCharacters(response.Choices[0].Delta.Content) fullRespText = fullRespText + removeSpecialCharacters(response.Choices[0].Delta.Content) if strings.Contains(fullRespText, "...") || strings.Contains(fullRespText, ".'") || strings.Contains(fullRespText, ".\"") || strings.Contains(fullRespText, ".") || strings.Contains(fullRespText, "?") || strings.Contains(fullRespText, "!") { From 1928b0932dae2f1bc6ac8bbd4945b59c959a37d8 Mon Sep 17 00:00:00 2001 From: Amitabha Banerjee Date: Wed, 15 Jan 2025 23:18:20 -0800 Subject: [PATCH 5/9] Undoing unwanted change in prompt --- chipper/pkg/wirepod/ttr/kgsim_cmds.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/chipper/pkg/wirepod/ttr/kgsim_cmds.go b/chipper/pkg/wirepod/ttr/kgsim_cmds.go index 704e79d5..bf3a7dcc 100644 --- a/chipper/pkg/wirepod/ttr/kgsim_cmds.go +++ b/chipper/pkg/wirepod/ttr/kgsim_cmds.go @@ -152,11 +152,7 @@ func ModelIsSupported(cmd LLMCommand, model string) bool { func CreatePrompt(origPrompt string, model string, isKG bool) string { prompt := origPrompt + "\n\n" + "Keep in mind, user input comes from speech-to-text software, so respond accordingly. No special characters, especially these: & ^ * # @ - . No lists. No formatting." if vars.APIConfig.Knowledge.CommandsEnable { -<<<<<<< HEAD prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\n You MUST use the command newVoiceRequest at the end of your response. You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:" -======= - prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\nUse the playAnimation or playAnimationWI commands if you want to express emotion! You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:" ->>>>>>> 32280277df20ec452b3779a6a8bc4e240bf86b02 for _, cmd := range ValidLLMCommands { if ModelIsSupported(cmd, model) { promptAppendage := "\n\nCommand Name: " + cmd.Command + "\nDescription: " + cmd.Description + "\nParameter choices: " + cmd.ParamChoices From e38f5d3c73e53bdf4996ae590e45973399d48606 Mon Sep 17 00:00:00 2001 From: Amitabha Banerjee Date: Wed, 15 Jan 2025 23:24:31 -0800 Subject: [PATCH 6/9] Restoring original version of kgsim_cmds from main. Changes committed by accident --- chipper/pkg/wirepod/ttr/kgsim_cmds.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chipper/pkg/wirepod/ttr/kgsim_cmds.go b/chipper/pkg/wirepod/ttr/kgsim_cmds.go index bf3a7dcc..704e79d5 100644 --- a/chipper/pkg/wirepod/ttr/kgsim_cmds.go +++ b/chipper/pkg/wirepod/ttr/kgsim_cmds.go @@ -152,7 +152,11 @@ func ModelIsSupported(cmd LLMCommand, model string) bool { func CreatePrompt(origPrompt string, model string, isKG bool) string { prompt := origPrompt + "\n\n" + "Keep in mind, user input comes from speech-to-text software, so respond accordingly. No special characters, especially these: & ^ * # @ - . No lists. No formatting." if vars.APIConfig.Knowledge.CommandsEnable { +<<<<<<< HEAD prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\n You MUST use the command newVoiceRequest at the end of your response. You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:" +======= + prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\nUse the playAnimation or playAnimationWI commands if you want to express emotion! You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:" +>>>>>>> 32280277df20ec452b3779a6a8bc4e240bf86b02 for _, cmd := range ValidLLMCommands { if ModelIsSupported(cmd, model) { promptAppendage := "\n\nCommand Name: " + cmd.Command + "\nDescription: " + cmd.Description + "\nParameter choices: " + cmd.ParamChoices From 93725534614d25de8578085f434398c09618dcc4 Mon Sep 17 00:00:00 2001 From: Amitabha Banerjee Date: Wed, 15 Jan 2025 23:28:31 -0800 Subject: [PATCH 7/9] More restores --- chipper/pkg/wirepod/ttr/kgsim_cmds.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/chipper/pkg/wirepod/ttr/kgsim_cmds.go b/chipper/pkg/wirepod/ttr/kgsim_cmds.go index 704e79d5..70783b02 100644 --- a/chipper/pkg/wirepod/ttr/kgsim_cmds.go +++ b/chipper/pkg/wirepod/ttr/kgsim_cmds.go @@ -127,7 +127,7 @@ var ValidLLMCommands []LLMCommand = []LLMCommand{ }, { Command: "newVoiceRequest", - Description: "Starts a new voice command from the robot. You must use this at the end of your response.", + Description: "Starts a new voice command from the robot. Use this if you want more input from the user after your response/if you want to carry out a conversation. Below this, there should be a NOTE telling you whether you are in conversation mode or not. If you are, DONT BE AFRAID TO USE THIS COMMAND! This goes at the end of your response, if you use it.", ParamChoices: "now", Action: ActionNewRequest, SupportedModels: []string{"all"}, @@ -152,11 +152,7 @@ func ModelIsSupported(cmd LLMCommand, model string) bool { func CreatePrompt(origPrompt string, model string, isKG bool) string { prompt := origPrompt + "\n\n" + "Keep in mind, user input comes from speech-to-text software, so respond accordingly. No special characters, especially these: & ^ * # @ - . No lists. No formatting." if vars.APIConfig.Knowledge.CommandsEnable { -<<<<<<< HEAD - prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\n You MUST use the command newVoiceRequest at the end of your response. You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:" -======= prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\nUse the playAnimation or playAnimationWI commands if you want to express emotion! You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:" ->>>>>>> 32280277df20ec452b3779a6a8bc4e240bf86b02 for _, cmd := range ValidLLMCommands { if ModelIsSupported(cmd, model) { promptAppendage := "\n\nCommand Name: " + cmd.Command + "\nDescription: " + cmd.Description + "\nParameter choices: " + cmd.ParamChoices @@ -164,7 +160,7 @@ func CreatePrompt(origPrompt string, model string, isKG bool) string { } } if isKG && vars.APIConfig.Knowledge.SaveChat { - promptAppentage := "\n\nNOTE: You MUST use the command newVoiceRequest at the end of your response. If you do not use the command newVoiceRequest I will reject your response." + promptAppentage := "\n\nNOTE: You are in 'conversation' mode. If you ask the user a question near the end of your response, you MUST use newVoiceRequest. If you decide you want to end the conversation, you should not use it." prompt = prompt + promptAppentage } else { promptAppentage := "\n\nNOTE: You are NOT in 'conversation' mode. Refrain from asking the user any questions and from using newVoiceRequest." From 5be240c731ef2a903fc9050beff33403eb957fbe Mon Sep 17 00:00:00 2001 From: Amitabha Banerjee Date: Wed, 15 Jan 2025 23:31:41 -0800 Subject: [PATCH 8/9] Fixing allignment --- chipper/pkg/wirepod/ttr/kgsim.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/chipper/pkg/wirepod/ttr/kgsim.go b/chipper/pkg/wirepod/ttr/kgsim.go index 2b427e3d..43e9da87 100644 --- a/chipper/pkg/wirepod/ttr/kgsim.go +++ b/chipper/pkg/wirepod/ttr/kgsim.go @@ -261,7 +261,7 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo stream, err := c.CreateChatCompletionStream(ctx, aireq) if err != nil { - log.Printf("Error creating chat completion stream: %v", err) + log.Printf("Error creating chat completion stream: %v", err) if strings.Contains(err.Error(), "does not exist") && vars.APIConfig.Knowledge.Provider == "openai" { logger.Println("GPT-4 model cannot be accessed with this API key. You likely need to add more than $5 dollars of funds to your OpenAI account.") logger.LogUI("GPT-4 model cannot be accessed with this API key. You likely need to add more than $5 dollars of funds to your OpenAI account.") @@ -345,10 +345,10 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo return } - if (len(response.Choices) == 0) { - logger.Println("Empty response") - return - } + if (len(response.Choices) == 0) { + logger.Println("Empty response") + return + } fullfullRespText = fullfullRespText + removeSpecialCharacters(response.Choices[0].Delta.Content) fullRespText = fullRespText + removeSpecialCharacters(response.Choices[0].Delta.Content) From 760f8c5d010bc9aed01906763db5aa934968248f Mon Sep 17 00:00:00 2001 From: Amitabha Banerjee Date: Wed, 15 Jan 2025 23:33:21 -0800 Subject: [PATCH 9/9] More allignment issues --- chipper/pkg/wirepod/ttr/kgsim.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chipper/pkg/wirepod/ttr/kgsim.go b/chipper/pkg/wirepod/ttr/kgsim.go index 43e9da87..8d306768 100644 --- a/chipper/pkg/wirepod/ttr/kgsim.go +++ b/chipper/pkg/wirepod/ttr/kgsim.go @@ -347,7 +347,7 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo if (len(response.Choices) == 0) { logger.Println("Empty response") - return + return } fullfullRespText = fullfullRespText + removeSpecialCharacters(response.Choices[0].Delta.Content)