Merge pull request #217 from sj-distributor/enhance-ai-speech-session

Enhance ai speech assistant session

Author: happyshower

src/SmartTalk.Core/DbUpFile/Scripts_2025/Script00019_add_custom_record_analyze_prompt_for_ai_assisatnt_tabl.sql src/SmartTalk.Core/DbUpFile/Scripts_2025/Script0019_add_custom_record_analyze_prompt_for_ai_assisatnt_tabl.sql

@@ -0,0 +1 @@
+alter table `ai_speech_assistant_function_call` add column `type` int not null default 0;

src/SmartTalk.Core/DbUpFile/Scripts_2025/Script0022_modify_ai_speech_assistant_function_call_table.sql

@@ -1,5 +1,6 @@
 using System.ComponentModel.DataAnnotations;
 using System.ComponentModel.DataAnnotations.Schema;
+using SmartTalk.Messages.Enums.AiSpeechAssistant;
 
 namespace SmartTalk.Core.Domain.AISpeechAssistant;
 
@@ -20,6 +21,9 @@ public class AiSpeechAssistantFunctionCall : IEntity, IHasCreatedFields
     [Column("content")]
     public string Content { get; set; }
 
+    [Column("type")]
+    public AiSpeechAssistantSessionConfigType Type { get; set; }
+    
     [Column("created_date")]
     public DateTimeOffset CreatedDate { get; set; }
 }

src/SmartTalk.Core/Domain/AISpeechAssistant/AiSpeechAssistantFunctionCall.cs

@@ -729,32 +729,39 @@ private async Task SendToWebSocketAsync(WebSocket socket, object message)
 
     private async Task SendSessionUpdateAsync(WebSocket openAiWebSocket, Domain.AISpeechAssistant.AiSpeechAssistant assistant, string prompt)
     {
-        var tools = await InitialSessionToolsAsync(assistant).ConfigureAwait(false);
+        var configs = await InitialSessionConfigAsync(assistant).ConfigureAwait(false);
 
         var sessionUpdate = new
         {
             type = "session.update",
             session = new
             {
-                turn_detection = new { type = "server_vad" },
+                turn_detection = InitialSessionTurnDirection(configs),
                 input_audio_format = "g711_ulaw",
                 output_audio_format = "g711_ulaw",
                 voice = string.IsNullOrEmpty(assistant.Voice) ? "alloy" : assistant.Voice,
                 instructions = prompt,
                 modalities = new[] { "text", "audio" },
                 temperature = 0.8,
                 input_audio_transcription = new { model = "whisper-1" },
-                tools = tools
+                tools = configs.Where(x => x.Type == AiSpeechAssistantSessionConfigType.Tool).Select(x => x.Config)
             }
         };
 
         await SendToWebSocketAsync(openAiWebSocket, sessionUpdate);
     }
 
-    private async Task<IEnumerable<OpenAiRealtimeToolDto>> InitialSessionToolsAsync(Domain.AISpeechAssistant.AiSpeechAssistant assistant, CancellationToken cancellationToken = default)
+    private async Task<List<(AiSpeechAssistantSessionConfigType Type, object Config)>> InitialSessionConfigAsync(Domain.AISpeechAssistant.AiSpeechAssistant assistant, CancellationToken cancellationToken = default)
     {
         var functions = await _aiSpeechAssistantDataProvider.GetAiSpeechAssistantFunctionCallByAssistantIdAsync(assistant.Id, cancellationToken).ConfigureAwait(false);
 
-        return functions.Count == 0 ? [] : functions.Where(x => !string.IsNullOrWhiteSpace(x.Content)).Select(x => JsonConvert.DeserializeObject<OpenAiRealtimeToolDto>(x.Content));
+        return functions.Count == 0 ? [] : functions.Where(x => !string.IsNullOrWhiteSpace(x.Content)).Select(x => (x.Type, JsonConvert.DeserializeObject<object>(x.Content))).ToList();
     }
+
+    private object InitialSessionTurnDirection(List<(AiSpeechAssistantSessionConfigType Type, object Config)> configs)
+    {
+        var turnDetection = configs.FirstOrDefault(x => x.Type == AiSpeechAssistantSessionConfigType.TurnDirection);
+
+        return turnDetection.Config ?? new { type = "server_vad" };
+    } 
 }

src/SmartTalk.Core/Services/AiSpeechAssistant/AiSpeechAssistantService.cs

@@ -0,0 +1,7 @@
+namespace SmartTalk.Messages.Enums.AiSpeechAssistant;
+
+public enum AiSpeechAssistantSessionConfigType
+{
+    Tool,
+    TurnDirection
+}