Merge pull request #201 from sj-distributor/external-system-use-ai-speech-assistant

Support external system use ai speech assistant

Author: MonesyH

src/SmartTalk.Api/appsettings.json

@@ -78,7 +78,8 @@
   },
   "Smarties": {
     "BaseUrl": "",
-    "ApiKey": ""
+    "ApiKey": "",
+    "AiSpeechAssistantCallBackUrl": ""
   },
   "TranscriptionCallback": {
     "AuthHeaders": "",

src/SmartTalk.Core/DbUpFile/Scripts_2025/Script00019_add_custom_record_analyze_prompt_for_ai_assisatnt_tabl.sql

@@ -0,0 +1,2 @@
+alter table `ai_speech_assistant` add column `custom_record_analyze_prompt` text null;
+alter table `agent` add column `source_system` int not null default 0;

src/SmartTalk.Core/Domain/AISpeechAssistant/AiSpeechAssistant.cs

@@ -33,6 +33,9 @@ public class AiSpeechAssistant : IEntity, IHasCreatedFields
     [Column("greetings"), StringLength(1024)]
     public string Greetings { get; set; }
 
+    [Column("custom_record_analyze_prompt")]
+    public string CustomRecordAnalyzePrompt { get; set; }
+    
     [Column("created_date")]
     public DateTimeOffset CreatedDate { get; set; }
 }

src/SmartTalk.Core/Domain/System/Agent.cs

@@ -1,6 +1,7 @@
 using SmartTalk.Messages.Dto.Agent;
 using System.ComponentModel.DataAnnotations;
 using System.ComponentModel.DataAnnotations.Schema;
+using SmartTalk.Messages.Enums.Agent;
 
 namespace SmartTalk.Core.Domain.System;
 
@@ -21,6 +22,9 @@ public class Agent : IEntity
     [Column("type")]
     public AgentType Type { get; set; }
 
+    [Column("source_system")]
+    public AgentSourceSystem SourceSystem { get; set; }
+    
     [Column("created_date")]
     public DateTimeOffset CreatedDate { get; set; }
 }

src/SmartTalk.Core/Services/AiSpeechAssistant/AiSpeechAssistantService.cs

@@ -15,6 +15,7 @@
 using OpenAI.Chat;
 using SmartTalk.Core.Services.Agents;
 using SmartTalk.Core.Services.Http;
+using SmartTalk.Core.Services.Http.Clients;
 using SmartTalk.Messages.Constants;
 using SmartTalk.Core.Services.Jobs;
 using SmartTalk.Core.Services.PhoneOrder;
@@ -30,6 +31,7 @@
 using SmartTalk.Messages.Commands.AiSpeechAssistant;
 using SmartTalk.Messages.Commands.PhoneOrder;
 using SmartTalk.Messages.Dto.Agent;
+using SmartTalk.Messages.Enums.Agent;
 using SmartTalk.Messages.Enums.PhoneOrder;
 using JsonSerializer = System.Text.Json.JsonSerializer;
 using RecordingResource = Twilio.Rest.Api.V2010.Account.Call.RecordingResource;
@@ -56,8 +58,8 @@ public class AiSpeechAssistantService : IAiSpeechAssistantService
     private readonly IMapper _mapper;
     private readonly OpenAiSettings _openAiSettings;
     private readonly TwilioSettings _twilioSettings;
+    private readonly ISmartiesClient _smartiesClient;
     private readonly ZhiPuAiSettings _zhiPuAiSettings;
-    private readonly IAgentDataProvider _agentDataProvider;
     private readonly IPhoneOrderService _phoneOrderService;
     private readonly ISmartTalkHttpClientFactory _httpClientFactory;
     private readonly IPhoneOrderDataProvider _phoneOrderDataProvider;
@@ -68,8 +70,8 @@ public AiSpeechAssistantService(
         IMapper mapper,
         OpenAiSettings openAiSettings,
         TwilioSettings twilioSettings,
+        ISmartiesClient smartiesClient,
         ZhiPuAiSettings zhiPuAiSettings,
-        IAgentDataProvider agentDataProvider,
         IPhoneOrderService phoneOrderService,
         ISmartTalkHttpClientFactory httpClientFactory,
         IPhoneOrderDataProvider phoneOrderDataProvider,
@@ -79,9 +81,9 @@ public AiSpeechAssistantService(
         _mapper = mapper;
         _openAiSettings = openAiSettings;
         _twilioSettings = twilioSettings;
+        _smartiesClient = smartiesClient;
         _zhiPuAiSettings = zhiPuAiSettings;
         _phoneOrderService = phoneOrderService;
-        _agentDataProvider = agentDataProvider;
         _httpClientFactory = httpClientFactory;
         _backgroundJobClient = backgroundJobClient;
         _phoneOrderDataProvider = phoneOrderDataProvider;
@@ -153,21 +155,21 @@ public async Task ReceivePhoneRecordingStatusCallbackAsync(ReceivePhoneRecording
     {
         Log.Information("Handling receive phone record: {@command}", command);
 
-        var record = await _phoneOrderDataProvider.GetPhoneOrderRecordBySessionIdAsync(command.CallSid, cancellationToken).ConfigureAwait(false);
-
+        var (record, agent, aiSpeechAssistant) = await _phoneOrderDataProvider.GetRecordWithAgentAndAssistantAsync(command.CallSid, cancellationToken).ConfigureAwait(false);
+        
         Log.Information("Get phone order record: {@record}", record);
 
         record.Url = command.RecordingUrl;
         record.Status = PhoneOrderRecordStatus.Sent;
-
-        var agent = await _agentDataProvider.GetAgentByIdAsync(record.AgentId, cancellationToken: cancellationToken).ConfigureAwait(false);
-
+        
         ChatClient client = new("gpt-4o-audio-preview", _openAiSettings.ApiKey);
         var audioFileRawBytes = await _httpClientFactory.GetAsync<byte[]>(record.Url, cancellationToken).ConfigureAwait(false);
         var audioData = BinaryData.FromBytes(audioFileRawBytes);
         List<ChatMessage> messages =
         [
-            new SystemChatMessage("你是一名電話錄音的分析員，通過聽取錄音內容和語氣情緒作出精確分析，冩出一份分析報告。\n\n分析報告的格式：交談主題：xxx\n\n 內容摘要:xxx \n\n 客人情感與情緒: xxx \n\n 待辦事件: \n1.xxx\n2.xxx \n\n 客人下單內容(如果沒有則忽略)：1. 牛肉(1箱)\n2.雞腿肉(1箱)"),
+            new SystemChatMessage(string.IsNullOrEmpty(aiSpeechAssistant?.CustomRecordAnalyzePrompt)
+                ? "你是一名電話錄音的分析員，通過聽取錄音內容和語氣情緒作出精確分析，冩出一份分析報告。\n\n分析報告的格式：交談主題：xxx\n\n 內容摘要:xxx \n\n 客人情感與情緒: xxx \n\n 待辦事件: \n1.xxx\n2.xxx \n\n 客人下單內容(如果沒有則忽略)：1. 牛肉(1箱)\n2.雞腿肉(1箱)" 
+                : aiSpeechAssistant.CustomRecordAnalyzePrompt),
             new UserChatMessage(ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Wav)),
             new UserChatMessage("幫我根據錄音生成分析報告：")
         ];
@@ -178,6 +180,9 @@ public async Task ReceivePhoneRecordingStatusCallbackAsync(ReceivePhoneRecording
         Log.Information("sales record analyze report:" + completion.Content.FirstOrDefault()?.Text);
         record.TranscriptionText = completion.Content.FirstOrDefault()?.Text;
 
+        if (agent.SourceSystem == AgentSourceSystem.Smarties)
+            await _smartiesClient.CallBackSmartiesAiSpeechAssistantRecordAsync(new AiSpeechAssistantCallBackRequestDto { CallSid = command.CallSid, RecordUrl = record.Url, RecordAnalyzeReport =  record.TranscriptionText }, cancellationToken).ConfigureAwait(false);
+        
         if (!string.IsNullOrEmpty(agent.WechatRobotKey))
             await _phoneOrderService.SendWorkWeChatRobotNotifyAsync(audioFileRawBytes, agent.WechatRobotKey, "錄音分析報告：\n" + record.TranscriptionText, cancellationToken).ConfigureAwait(false);
 

src/SmartTalk.Core/Services/Http/Clients/SmartiesClient.cs

@@ -2,6 +2,7 @@
 using SmartTalk.Core.Ioc;
 using Smarties.Messages.Requests.Ask;
 using SmartTalk.Core.Settings.Smarties;
+using SmartTalk.Messages.Dto.AiSpeechAssistant;
 using SmartTalk.Messages.Dto.Smarties;
 
 namespace SmartTalk.Core.Services.Http.Clients;
@@ -11,6 +12,8 @@ public interface ISmartiesClient : IScopedDependency
     Task<AskGptResponse> PerformQueryAsync(AskGptRequest request, CancellationToken cancellationToken);
 
     Task<AskGptEmbeddingResponseDto> GetEmbeddingAsync(AskGptEmbeddingRequestDto request, CancellationToken cancellationToken);
+
+    Task CallBackSmartiesAiSpeechAssistantRecordAsync(AiSpeechAssistantCallBackRequestDto request, CancellationToken cancellationToken);
 }
 
 public class SmartiesClient : ISmartiesClient
@@ -46,4 +49,9 @@ public async Task<AskGptEmbeddingResponseDto> GetEmbeddingAsync(AskGptEmbeddingR
 
         return response;
     }
+
+    public async Task CallBackSmartiesAiSpeechAssistantRecordAsync(AiSpeechAssistantCallBackRequestDto request, CancellationToken cancellationToken)
+    {
+        await _httpClientFactory.PostAsJsonAsync(_smartiesSettings.AiSpeechAssistantCallBackUrl, request, cancellationToken, headers: _headers).ConfigureAwait(false);
+    }
 }

src/SmartTalk.Core/Services/PhoneOrder/PhoneOrderDataProvider.Record.cs

@@ -33,6 +33,8 @@ Task<List<GetPhoneOrderRecordsWithUserCountDto>> GetPhoneOrderRecordsWithUserCou
     Task<PhoneOrderRecord> GetPhoneOrderRecordByIdAsync(int recordId, CancellationToken cancellationToken);
 
     Task<PhoneOrderRecord> GetPhoneOrderRecordBySessionIdAsync(string sessionId, CancellationToken cancellationToken);
+    
+    Task<(PhoneOrderRecord, Agent, Domain.AISpeechAssistant.AiSpeechAssistant)> GetRecordWithAgentAndAssistantAsync(string sessionId, CancellationToken cancellationToken);
 }
 
 public partial class PhoneOrderDataProvider
@@ -177,4 +179,19 @@ public async Task<PhoneOrderRecord> GetPhoneOrderRecordBySessionIdAsync(string s
     {
         return await _repository.Query<PhoneOrderRecord>().Where(x => x.SessionId == sessionId).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
     }
+    
+    public async Task<(PhoneOrderRecord, Agent, Domain.AISpeechAssistant.AiSpeechAssistant)> GetRecordWithAgentAndAssistantAsync(string sessionId, CancellationToken cancellationToken)
+    {
+        var result = await (
+            from record in _repository.Query<PhoneOrderRecord>()
+            where record.SessionId == sessionId
+            join agent in _repository.Query<Agent>() on record.AgentId equals agent.Id into agentGroup
+            from agent in agentGroup.DefaultIfEmpty()
+            join assistant in _repository.Query<Domain.AISpeechAssistant.AiSpeechAssistant>() on record.AgentId equals assistant.AgentId into assistantGroup
+            from assistant in assistantGroup.DefaultIfEmpty()
+            select new { record, agent, assistant }
+        ).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
+
+        return (result?.record, result?.agent, result?.assistant);
+    }
 }

src/SmartTalk.Core/Settings/Smarties/SmartiesSetting.cs

@@ -9,9 +9,13 @@ public SmartiesSetting(IConfiguration configuration)
         BaseUrl = configuration.GetValue<string>("Smarties:BaseUrl");
 
         ApiKey = configuration.GetValue<string>("Smarties:ApiKey");
+
+        AiSpeechAssistantCallBackUrl = configuration.GetValue<string>("Smarties:AiSpeechAssistantCallBackUrl");
     }
 
     public string BaseUrl { get; set; }
 
     public string ApiKey { get; set; }
+    
+    public string AiSpeechAssistantCallBackUrl { get; set; }
 }

src/SmartTalk.Messages/Dto/Agent/AgentDto.cs

@@ -1,3 +1,5 @@
+using SmartTalk.Messages.Enums.Agent;
+
 namespace SmartTalk.Messages.Dto.Agent;
 
 public class AgentDto
@@ -10,5 +12,7 @@ public class AgentDto
 
     public AgentType Type { get; set; }
 
+    public AgentSourceSystem SourceSystem { get; set; }
+    
     public DateTimeOffset CreatedDate { get; set; }
 }

src/SmartTalk.Messages/Dto/AiSpeechAssistant/AiSpeechAssistantCallBackRequestDto.cs

@@ -0,0 +1,10 @@
+namespace SmartTalk.Messages.Dto.AiSpeechAssistant;
+
+public class AiSpeechAssistantCallBackRequestDto
+{
+    public string CallSid { get; set; }
+    
+    public string RecordUrl { get; set; }
+    
+    public string RecordAnalyzeReport { get; set; }
+}

src/SmartTalk.Messages/Dto/AiSpeechAssistant/AiSpeechAssistantDto.cs

@@ -20,5 +20,7 @@ public class AiSpeechAssistantDto
 
     public string Greetings { get; set; }
 
+    public string CustomRecordAnalyzePrompt { get; set; }
+    
     public DateTimeOffset CreatedDate { get; set; }
 }

src/SmartTalk.Messages/Enums/Agent/AgentSourceSystem.cs

@@ -0,0 +1,7 @@
+namespace SmartTalk.Messages.Enums.Agent;
+
+public enum AgentSourceSystem
+{
+    Self,
+    Smarties
+}