Merge pull request #200 from sj-distributor/openai-audio-api

Implement Openai audio api

Author: MonesyH

src/SmartTalk.Api/Startup.cs

@@ -1,5 +1,4 @@
 using Serilog;
-using OpenAI.Extensions;
 using SmartTalk.Messages;
 using Correlate.AspNetCore;
 using SmartTalk.Api.Filters;
@@ -39,8 +38,6 @@ public void ConfigureServices(IServiceCollection services)
         });
 
         services.AddHangfireInternal(Configuration);
-        services.AddOpenAIService(settings => { settings.ApiKey = new OpenAiSettings(Configuration).ApiKey; })
-            .ConfigureHttpClient(http => http.Timeout = TimeSpan.FromSeconds(6000));
     }
 
     public void Configure(IApplicationBuilder app, IWebHostEnvironment env)

src/SmartTalk.Core/DbUpFile/Scripts_2025/Script00016_add_robot_url_for_agent_tabl.sql

@@ -0,0 +1 @@
+alter table `agent` add column `wechat_robot_url` varchar(256) null;

src/SmartTalk.Core/DbUpFile/Scripts_2025/Script00017_add_robot_upload_url_for_agent_tabl.sql

@@ -0,0 +1 @@
+ALTER TABLE `agent` CHANGE `wechat_robot_url` `wechat_robot_key` VARCHAR(255) null;

src/SmartTalk.Core/DbUpFile/Scripts_2025/Script00018_add_voice_for_ai_assisatnt_tabl.sql

@@ -0,0 +1 @@
+alter table `ai_speech_assistant` add column `voice` varchar(36) null;

src/SmartTalk.Core/Domain/AISpeechAssistant/AiSpeechAssistant.cs

@@ -21,6 +21,9 @@ public class AiSpeechAssistant : IEntity, IHasCreatedFields
     [Column("url"), StringLength(512)]
     public string Url { get; set; }
 
+    [Column("voice"), StringLength(36)]
+    public string Voice { get; set; }
+    
     [Column("provider")]
     public AiSpeechAssistantProvider Provider { get; set; }
 

src/SmartTalk.Core/Domain/System/Agent.cs

@@ -11,6 +11,9 @@ public class Agent : IEntity
     [Column("id")]
     [DatabaseGenerated(DatabaseGeneratedOption.Identity)]
     public int Id { get; set; }
+    
+    [Column("wechat_robot_key"), StringLength(256)]
+    public string WechatRobotKey { get; set; }
 
     [Column("relate_id")]
     public int RelateId { get; set; }

src/SmartTalk.Core/Mappings/SttMapping.cs

@@ -11,6 +11,8 @@ namespace SmartTalk.Core.Services.Agents;
 public interface IAgentDataProvider : IScopedDependency
 {
     Task<Agent> GetAgentAsync(int? id = null, AgentType? type = null, int? relateId = null, string name = null, CancellationToken cancellationToken = default);
+    
+    Task<Agent> GetAgentByIdAsync(int id, CancellationToken cancellationToken);
 
     Task<List<Agent>> GetAgentsAsync(AgentType type, CancellationToken cancellationToken);
 }
@@ -47,7 +49,12 @@ join restaurant in _repository.Query<Restaurant>() on agent.RelateId equals rest
 
         return query == null ? null : await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
     }
-    
+
+    public async Task<Agent> GetAgentByIdAsync(int id, CancellationToken cancellationToken)
+    {
+        return await _repository.Query<Agent>().Where(x => x.Id == id).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
+    }
+
     public async Task<List<Agent>> GetAgentsAsync(AgentType type, CancellationToken cancellationToken)
     {
         return await _repository.Query<Agent>().Where(x => x.Type == type).ToListAsync(cancellationToken).ConfigureAwait(false);

src/SmartTalk.Core/Services/Agents/AgentDataProvider.cs

@@ -12,6 +12,9 @@
 using AutoMapper;
 using SmartTalk.Core.Constants;
 using Microsoft.AspNetCore.Http;
+using OpenAI.Chat;
+using SmartTalk.Core.Services.Agents;
+using SmartTalk.Core.Services.Http;
 using SmartTalk.Messages.Constants;
 using SmartTalk.Core.Services.Jobs;
 using SmartTalk.Core.Services.PhoneOrder;
@@ -26,6 +29,7 @@
 using SmartTalk.Messages.Events.AiSpeechAssistant;
 using SmartTalk.Messages.Commands.AiSpeechAssistant;
 using SmartTalk.Messages.Commands.PhoneOrder;
+using SmartTalk.Messages.Dto.Agent;
 using SmartTalk.Messages.Enums.PhoneOrder;
 using JsonSerializer = System.Text.Json.JsonSerializer;
 using RecordingResource = Twilio.Rest.Api.V2010.Account.Call.RecordingResource;
@@ -53,6 +57,9 @@ public class AiSpeechAssistantService : IAiSpeechAssistantService
     private readonly OpenAiSettings _openAiSettings;
     private readonly TwilioSettings _twilioSettings;
     private readonly ZhiPuAiSettings _zhiPuAiSettings;
+    private readonly IAgentDataProvider _agentDataProvider;
+    private readonly IPhoneOrderService _phoneOrderService;
+    private readonly ISmartTalkHttpClientFactory _httpClientFactory;
     private readonly IPhoneOrderDataProvider _phoneOrderDataProvider;
     private readonly ISmartTalkBackgroundJobClient _backgroundJobClient;
     private readonly IAiSpeechAssistantDataProvider _aiSpeechAssistantDataProvider;
@@ -62,6 +69,9 @@ public AiSpeechAssistantService(
         OpenAiSettings openAiSettings,
         TwilioSettings twilioSettings,
         ZhiPuAiSettings zhiPuAiSettings,
+        IAgentDataProvider agentDataProvider,
+        IPhoneOrderService phoneOrderService,
+        ISmartTalkHttpClientFactory httpClientFactory,
         IPhoneOrderDataProvider phoneOrderDataProvider,
         ISmartTalkBackgroundJobClient backgroundJobClient,
         IAiSpeechAssistantDataProvider aiSpeechAssistantDataProvider)
@@ -70,6 +80,9 @@ public AiSpeechAssistantService(
         _openAiSettings = openAiSettings;
         _twilioSettings = twilioSettings;
         _zhiPuAiSettings = zhiPuAiSettings;
+        _phoneOrderService = phoneOrderService;
+        _agentDataProvider = agentDataProvider;
+        _httpClientFactory = httpClientFactory;
         _backgroundJobClient = backgroundJobClient;
         _phoneOrderDataProvider = phoneOrderDataProvider;
         _aiSpeechAssistantDataProvider = aiSpeechAssistantDataProvider;
@@ -146,6 +159,28 @@ public async Task ReceivePhoneRecordingStatusCallbackAsync(ReceivePhoneRecording
 
         record.Url = command.RecordingUrl;
         record.Status = PhoneOrderRecordStatus.Sent;
+
+        var agent = await _agentDataProvider.GetAgentByIdAsync(record.AgentId, cancellationToken: cancellationToken).ConfigureAwait(false);
+
+        ChatClient client = new("gpt-4o-audio-preview", _openAiSettings.ApiKey);
+        var audioFileRawBytes = await _httpClientFactory.GetAsync<byte[]>(record.Url, cancellationToken).ConfigureAwait(false);
+        var audioData = BinaryData.FromBytes(audioFileRawBytes);
+        List<ChatMessage> messages =
+        [
+            new SystemChatMessage("你是一名電話錄音的分析員，通過聽取錄音內容和語氣情緒作出精確分析，冩出一份分析報告。\n\n分析報告的格式：交談主題：xxx\n\n 內容摘要:xxx \n\n 客人情感與情緒: xxx \n\n 待辦事件: \n1.xxx\n2.xxx \n\n 客人下單內容(如果沒有則忽略)：1. 牛肉(1箱)\n2.雞腿肉(1箱)"),
+            new UserChatMessage(ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Wav)),
+            new UserChatMessage("幫我根據錄音生成分析報告：")
+        ];
+        
+        ChatCompletionOptions options = new() { ResponseModalities = ChatResponseModalities.Text };
+
+        ChatCompletion completion = await client.CompleteChatAsync(messages, options, cancellationToken);
+        Log.Information("sales record analyze report:" + completion.Content.FirstOrDefault()?.Text);
+        record.TranscriptionText = completion.Content.FirstOrDefault()?.Text;
+
+        if (!string.IsNullOrEmpty(agent.WechatRobotKey))
+            await _phoneOrderService.SendWorkWeChatRobotNotifyAsync(audioFileRawBytes, agent.WechatRobotKey, "錄音分析報告：\n" + record.TranscriptionText, cancellationToken).ConfigureAwait(false);
+
         await _phoneOrderDataProvider.UpdatePhoneOrderRecordsAsync(record, cancellationToken: cancellationToken).ConfigureAwait(false);
     }
 
@@ -200,7 +235,9 @@ private async Task<WebSocket> ConnectOpenAiRealTimeSocketAsync(Domain.AISpeechAs
         var url = string.IsNullOrEmpty(assistant.Url) ? AiSpeechAssistantStore.DefaultUrl : assistant.Url;
 
         await openAiWebSocket.ConnectAsync(new Uri(url), cancellationToken).ConfigureAwait(false);
+
         await SendSessionUpdateAsync(openAiWebSocket, assistant, prompt).ConfigureAwait(false);
+
         return openAiWebSocket;
     }
 
@@ -401,7 +438,7 @@ private async Task SendToTwilioAsync(WebSocket twilioWebSocket, WebSocket openAi
                         }
                     }
 
-                    if (!context.InitialConversationSent)
+                    if (!context.InitialConversationSent && !string.IsNullOrEmpty(context.Assistant.Greetings))
                     {
                         await SendInitialConversationItem(openAiWebSocket, context);
                         context.InitialConversationSent = true;
@@ -697,7 +734,7 @@ private async Task SendSessionUpdateAsync(WebSocket openAiWebSocket, Domain.AISp
                 turn_detection = new { type = "server_vad" },
                 input_audio_format = "g711_ulaw",
                 output_audio_format = "g711_ulaw",
-                voice = "alloy",
+                voice = string.IsNullOrEmpty(assistant.Voice) ? "alloy" : assistant.Voice,
                 instructions = prompt,
                 modalities = new[] { "text", "audio" },
                 temperature = 0.8,

src/SmartTalk.Core/Services/AiSpeechAssistant/AiSpeechAssistantService.cs

@@ -1,15 +1,14 @@
 using Serilog;
 using AutoMapper;
 using System.Text;
-using OpenAI.Interfaces;
+using OpenAI.Audio;
 using SmartTalk.Core.Ioc;
-using OpenAI.ObjectModels;
 using SmartTalk.Core.Extensions;
 using SmartTalk.Messages.Dto.STT;
 using SmartTalk.Messages.Enums.STT;
 using SmartTalk.Core.Services.Http;
 using SmartTalk.Core.Services.Ffmpeg;
-using OpenAI.ObjectModels.RequestModels;
+using SmartTalk.Core.Settings.OpenAi;
 
 namespace SmartTalk.Core.Services.STT;
 
@@ -24,14 +23,14 @@ public class SpeechToTextService : ISpeechToTextService
 {
     private readonly IMapper _mapper;
     private readonly IFfmpegService _ffmpegService;
-    private readonly IOpenAIService _openAiService;
+    private readonly OpenAiSettings _openAiSettings;
     private readonly ISmartiesHttpClientFactory _httpClientFactory;
 
-    public SpeechToTextService(IMapper mapper, IFfmpegService ffmpegService, IOpenAIService openAiService, ISmartiesHttpClientFactory httpClientFactory)
+    public SpeechToTextService(IMapper mapper, IFfmpegService ffmpegService, OpenAiSettings openAiSettings, ISmartiesHttpClientFactory httpClientFactory)
     {
         _mapper = mapper;
         _ffmpegService = ffmpegService;
-        _openAiService = openAiService;
+        _openAiSettings = openAiSettings;
         _httpClientFactory = httpClientFactory;
     }
 
@@ -51,52 +50,45 @@ public async Task<string> SpeechToTextAsync(
         {
             var transcriptionResponse = await TranscriptionAsync(audio, language, fileType, responseFormat, prompt, cancellationToken).ConfigureAwait(false);
 
-            transcriptionResult.Append(transcriptionResponse.Text);
+            transcriptionResult.Append(transcriptionResponse);
         }
 
         Log.Information("Transcription result {Transcription}", transcriptionResult.ToString());
 
         return transcriptionResult.ToString();
     }
 
-    public async Task<AudioTranscriptionResponseDto> TranscriptionAsync(
+    public async Task<string> TranscriptionAsync(
         byte[] file, TranscriptionLanguage? language, TranscriptionFileType fileType = TranscriptionFileType.Wav, 
         TranscriptionResponseFormat responseFormat = TranscriptionResponseFormat.Vtt, string prompt = null, CancellationToken cancellationToken = default)
     {
+        AudioClient client = new("whisper-1", _openAiSettings.ApiKey);
+        
         var filename = $"{Guid.NewGuid()}.{fileType.ToString().ToLower()}";
 
         var fileResponseFormat = responseFormat switch
         {
-            TranscriptionResponseFormat.Vtt => StaticValues.AudioStatics.ResponseFormat.Vtt,
-            TranscriptionResponseFormat.Srt => StaticValues.AudioStatics.ResponseFormat.Srt,
-            TranscriptionResponseFormat.Text => StaticValues.AudioStatics.ResponseFormat.Text,
-            TranscriptionResponseFormat.Json => StaticValues.AudioStatics.ResponseFormat.Json,
-            TranscriptionResponseFormat.VerboseJson => StaticValues.AudioStatics.ResponseFormat.Vtt
+            TranscriptionResponseFormat.Vtt => "vtt",
+            TranscriptionResponseFormat.Srt => "srt",
+            TranscriptionResponseFormat.Text => "text",
+            TranscriptionResponseFormat.Json => "json",
+            TranscriptionResponseFormat.VerboseJson => "verbose_json",
+            _ => "text"
         };
-
-        var transcriptionRequest = new AudioCreateTranscriptionRequest
+        var stream = new MemoryStream(file);
+        
+        AudioTranscriptionOptions options = new()
         {
-            File = file,
-            FileName = filename,
-            Model = Models.WhisperV1,
-            ResponseFormat = fileResponseFormat
+            ResponseFormat = AudioTranscriptionFormat.Text,
+            Prompt = prompt
         };
-
-        if (language.HasValue) transcriptionRequest.Language = language.Value.GetDescription();
 
-        var response = await _httpClientFactory.SafelyProcessRequestAsync(nameof(SpeechToTextAsync), async () =>
-            await _openAiService.Audio.CreateTranscription(new AudioCreateTranscriptionRequest
-            {
-                File = file,
-                FileName = filename,
-                Model = Models.WhisperV1,
-                ResponseFormat = fileResponseFormat,
-                Language = language?.GetDescription(),
-                Prompt = prompt
-            }, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false);
+        if (language.HasValue) options.Language = language.Value.GetDescription();
+        
+        var response = await client.TranscribeAudioAsync(stream, "test.wav", options, cancellationToken);
 
         Log.Information("Transcription {FileName} response {@Response}", filename, response);
 
-        return _mapper.Map<AudioTranscriptionResponseDto>(response);
+        return response?.Value?.Text;
     }
 }

src/SmartTalk.Core/Services/STT/SpeechToTextService.cs

@@ -12,7 +12,6 @@
         <PackageReference Include="Autofac.Extensions.DependencyInjection" Version="8.0.0" />
         <PackageReference Include="AutoMapper" Version="12.0.1" />
         <PackageReference Include="AutoMapper.Contrib.Autofac.DependencyInjection" Version="7.1.0" />
-        <PackageReference Include="Betalgo.OpenAI" Version="8.5.1" />
         <PackageReference Include="dbup" Version="5.0.8" />
         <PackageReference Include="dbup-mysql" Version="5.0.10" />
         <PackageReference Include="Destructurama.JsonNet" Version="2.0.0" />
@@ -36,6 +35,7 @@
         <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
         <PackageReference Include="Microsoft.KernelMemory.AI.OpenAI" Version="0.35.240321.1" />
         <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+        <PackageReference Include="OpenAI" Version="2.2.0-beta.2" />
         <PackageReference Include="OpenCCNET" Version="1.0.2" />
         <PackageReference Include="NRedisStack" Version="0.12.0" />
         <PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="7.0.0" />

src/SmartTalk.Core/SmartTalk.Core.csproj

@@ -4,7 +4,6 @@
 using MySql.Data.MySqlClient;
 using Newtonsoft.Json;
 using NSubstitute;
-using OpenAI.Interfaces;
 using Serilog;
 using SmartTalk.Core;
 using SmartTalk.Core.DbUpFile;
@@ -46,7 +45,6 @@ private void RegisterBaseContainer(ContainerBuilder containerBuilder)
             new SmartTalkModule(logger, configuration, typeof(SmartTalkModule).Assembly, typeof(TestBase).Assembly));
 
         containerBuilder.RegisterInstance(new TestCurrentUser()).As<ICurrentUser>();
-        containerBuilder.RegisterInstance(Substitute.For<IOpenAIService>()).AsImplementedInterfaces();
         containerBuilder.RegisterInstance(Substitute.For<IHttpContextAccessor>()).AsImplementedInterfaces();
         containerBuilder.RegisterInstance(Substitute.For<IAliYunOssService>()).AsImplementedInterfaces();
         containerBuilder.RegisterInstance(Substitute.For<IEasyPosClient>()).AsImplementedInterfaces();

src/SmartTalk.IntegrationTests/TestBase.Initial.cs

@@ -6,6 +6,8 @@ public class AgentDto
 
     public int RelateId { get; set; }
 
+    public string WechatRobotKey { get; set; }
+    
     public AgentType Type { get; set; }
 
     public DateTimeOffset CreatedDate { get; set; }

src/SmartTalk.Messages/Dto/Agent/AgentDto.cs

@@ -12,6 +12,8 @@ public class AiSpeechAssistantDto
 
     public string Url { get; set; }
 
+    public string Voice { get; set; }
+    
     public AiSpeechAssistantProvider Provider { get; set; }
 
     public int AgentId { get; set; }