Skip to content

Commit 2a377cd

Browse files
MonesyHMonesyH
committed
Merge branch 'main' into fix-record-ai-phone-null-exception
2 parents 13e2a69 + cd8c633 commit 2a377cd

13 files changed

+154
-153
lines changed

‎src/SmartTalk.Core/Constants/OpenAiToolConstants.cs

+1
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,5 @@ public static class OpenAiToolConstants
1515
public const string CheckOrderStatus = "check_order_status";
1616
public const string RequestOrderDelivery = "request_order_delivery";
1717
public const string ConfirmCustomerInformation = "confirm_customer_name_phone";
18+
public const string ConfirmPickupTime = "confirm_pickup_time";
1819
}

‎src/SmartTalk.Core/DbUpFile/Scripts_2025/Script00014_add_order_comment_for_phone_order_record_table.sql

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
alter table `phone_order_record` add column `comments` varchar(1024) null;

‎src/SmartTalk.Core/DbUpFile/Scripts_2025/Script00015_add_ai_speech_assistant_function_call_table.sql

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
create table if not exists ai_speech_assistant_function_call
2+
(
3+
`id` int primary key auto_increment,
4+
`assistant_id` int not null,
5+
`name` varchar(255) not null,
6+
`content` text null,
7+
`created_date` datetime(3) not null
8+
) charset=utf8mb4;

‎src/SmartTalk.Core/Domain/AISpeechAssistant/AiSpeechAssistantFunctionCall.cs

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
using System.ComponentModel.DataAnnotations;
2+
using System.ComponentModel.DataAnnotations.Schema;
3+
4+
namespace SmartTalk.Core.Domain.AISpeechAssistant;
5+
6+
[Table("ai_speech_assistant_function_call")]
7+
public class AiSpeechAssistantFunctionCall : IEntity, IHasCreatedFields
8+
{
9+
[Key]
10+
[Column("id")]
11+
[DatabaseGenerated(DatabaseGeneratedOption.Identity)]
12+
public int Id { get; set; }
13+
14+
[Column("assistant_id")]
15+
public int AssistantId { get; set; }
16+
17+
[Column("name"), StringLength(255)]
18+
public string Name { get; set; }
19+
20+
[Column("content")]
21+
public string Content { get; set; }
22+
23+
[Column("created_date")]
24+
public DateTimeOffset CreatedDate { get; set; }
25+
}

‎src/SmartTalk.Core/Domain/PhoneOrder/PhoneOrderRecord.cs

+3
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ public class PhoneOrderRecord : IEntity
6060
[Column("customer_name"), StringLength(50)]
6161
public string CustomerName { get; set; }
6262

63+
[Column("comments")]
64+
public string Comments { get; set; }
65+
6366
[NotMapped]
6467
public UserAccount UserAccount { get; set; }
6568

‎src/SmartTalk.Core/Services/AiSpeechAssistant/AiSpeechAssistantDataProvider.cs

+8
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ public interface IAiSpeechAssistantDataProvider : IScopedDependency
1414
Task<Domain.AISpeechAssistant.AiSpeechAssistant> GetAiSpeechAssistantByNumbersAsync(string didNumber, CancellationToken cancellationToken);
1515

1616
Task<AiSpeechAssistantHumanContact> GetAiSpeechAssistantHumanContactByAssistantIdAsync(int assistantId, CancellationToken cancellationToken);
17+
18+
Task<List<AiSpeechAssistantFunctionCall>> GetAiSpeechAssistantFunctionCallByAssistantIdAsync(int assistantId, CancellationToken cancellationToken);
1719
}
1820

1921
public class AiSpeechAssistantDataProvider : IAiSpeechAssistantDataProvider
@@ -58,4 +60,10 @@ public async Task<AiSpeechAssistantHumanContact> GetAiSpeechAssistantHumanContac
5860
return await _repository.Query<AiSpeechAssistantHumanContact>().Where(x => x.AssistantId == assistantId)
5961
.FirstOrDefaultAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
6062
}
63+
64+
public async Task<List<AiSpeechAssistantFunctionCall>> GetAiSpeechAssistantFunctionCallByAssistantIdAsync(int assistantId, CancellationToken cancellationToken)
65+
{
66+
return await _repository.QueryNoTracking<AiSpeechAssistantFunctionCall>()
67+
.Where(x => x.AssistantId == assistantId).ToListAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
68+
}
6169
}

‎src/SmartTalk.Core/Services/AiSpeechAssistant/AiSpeechAssistantProcessJobService.cs

+3-3
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,10 @@ public async Task RecordAiSpeechAssistantCallAsync(AiSpeechAssistantStreamContex
6060
Tips = context.ConversationTranscription.FirstOrDefault().Item2,
6161
TranscriptionText = FormattedConversation(context.ConversationTranscription),
6262
Language = TranscriptionLanguage.Chinese,
63-
CreatedDate = callResource?.StartTime ?? DateTimeOffset.Now,
63+
CreatedDate = callResource.StartTime ?? DateTimeOffset.Now,
6464
OrderStatus = PhoneOrderOrderStatus.Pending,
65-
CustomerName = context.UserInfo?.UserName,
66-
PhoneNumber = context.UserInfo?.PhoneNumber
65+
CustomerName = context.UserInfo.UserName,
66+
PhoneNumber = context.UserInfo.PhoneNumber
6767
};
6868

6969
await _phoneOrderDataProvider.AddPhoneOrderRecordsAsync([record], cancellationToken: cancellationToken).ConfigureAwait(false);

‎src/SmartTalk.Core/Services/AiSpeechAssistant/AiSpeechAssistantService.cs

+35-143
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,7 @@ private async Task<WebSocket> ConnectOpenAiRealTimeSocketAsync(Domain.AISpeechAs
200200
var url = string.IsNullOrEmpty(assistant.Url) ? AiSpeechAssistantStore.DefaultUrl : assistant.Url;
201201

202202
await openAiWebSocket.ConnectAsync(new Uri(url), cancellationToken).ConfigureAwait(false);
203-
await SendSessionUpdateAsync(openAiWebSocket, prompt).ConfigureAwait(false);
203+
await SendSessionUpdateAsync(openAiWebSocket, assistant, prompt).ConfigureAwait(false);
204204
return openAiWebSocket;
205205
}
206206

@@ -374,6 +374,10 @@ private async Task SendToTwilioAsync(WebSocket twilioWebSocket, WebSocket openAi
374374
case OpenAiToolConstants.ConfirmCustomerInformation:
375375
await ProcessRecordCustomerInformationAsync(openAiWebSocket, context, outputElement, cancellationToken).ConfigureAwait(false);
376376
break;
377+
378+
case OpenAiToolConstants.ConfirmPickupTime:
379+
await ProcessRecordOrderPickupTimeAsync(openAiWebSocket, context, outputElement, cancellationToken).ConfigureAwait(false);
380+
break;
377381

378382
case OpenAiToolConstants.Hangup:
379383
await ProcessHangupAsync(openAiWebSocket, context, outputElement, cancellationToken).ConfigureAwait(false);
@@ -451,6 +455,25 @@ private async Task ProcessRecordCustomerInformationAsync(WebSocket openAiWebSock
451455
await SendToWebSocketAsync(openAiWebSocket, recordSuccess);
452456
await SendToWebSocketAsync(openAiWebSocket, new { type = "response.create" });
453457
}
458+
459+
private async Task ProcessRecordOrderPickupTimeAsync(WebSocket openAiWebSocket, AiSpeechAssistantStreamContextDto context, JsonElement jsonDocument, CancellationToken cancellationToken)
460+
{
461+
var recordSuccess = new
462+
{
463+
type = "conversation.item.create",
464+
item = new
465+
{
466+
type = "function_call_output",
467+
call_id = jsonDocument.GetProperty("call_id").GetString(),
468+
output = "Record the time when the customer pickup the order."
469+
}
470+
};
471+
472+
context.OrderItems.Comments = JsonConvert.DeserializeObject<AiSpeechAssistantOrderDto>(jsonDocument.GetProperty("arguments").ToString())?.Comments ?? string.Empty;
473+
474+
await SendToWebSocketAsync(openAiWebSocket, recordSuccess);
475+
await SendToWebSocketAsync(openAiWebSocket, new { type = "response.create" });
476+
}
454477

455478
private async Task ProcessHangupAsync(WebSocket openAiWebSocket, AiSpeechAssistantStreamContextDto context, JsonElement jsonDocument, CancellationToken cancellationToken)
456479
{
@@ -662,8 +685,10 @@ private async Task SendToWebSocketAsync(WebSocket socket, object message)
662685
await socket.SendAsync(new ArraySegment<byte>(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(message))), WebSocketMessageType.Text, true, CancellationToken.None);
663686
}
664687

665-
private async Task SendSessionUpdateAsync(WebSocket openAiWebSocket, string prompt)
688+
private async Task SendSessionUpdateAsync(WebSocket openAiWebSocket, Domain.AISpeechAssistant.AiSpeechAssistant assistant, string prompt)
666689
{
690+
var tools = await InitialSessionToolsAsync(assistant).ConfigureAwait(false);
691+
667692
var sessionUpdate = new
668693
{
669694
type = "session.update",
@@ -677,150 +702,17 @@ private async Task SendSessionUpdateAsync(WebSocket openAiWebSocket, string prom
677702
modalities = new[] { "text", "audio" },
678703
temperature = 0.8,
679704
input_audio_transcription = new { model = "whisper-1" },
680-
tools = new[]
681-
{
682-
new OpenAiRealtimeToolDto
683-
{
684-
Type = "function",
685-
Name = OpenAiToolConstants.TransferCall,
686-
Description = "Triggered when the customer requests to transfer the call to a real person, or when the customer is not satisfied with the current answer and wants someone else to serve him/her"
687-
},
688-
new OpenAiRealtimeToolDto
689-
{
690-
Type = "function",
691-
Name = OpenAiToolConstants.HandlePhoneOrderIssues,
692-
Description = "Resolve inquiries or issues related to orders placed via phone."
693-
},
694-
new OpenAiRealtimeToolDto
695-
{
696-
Type = "function",
697-
Name = OpenAiToolConstants.HandleThirdPartyDelayedDelivery,
698-
Description = "Address delayed delivery issues for orders placed through third-party platforms."
699-
},
700-
new OpenAiRealtimeToolDto
701-
{
702-
Type = "function",
703-
Name = OpenAiToolConstants.HandleThirdPartyPickupTimeChange,
704-
Description = "Manage pickup time modification requests for orders placed through third-party platforms."
705-
},
706-
new OpenAiRealtimeToolDto
707-
{
708-
Type = "function",
709-
Name = OpenAiToolConstants.HandleThirdPartyFoodQuality,
710-
Description = "Resolve food quality or taste complaints for orders placed through third-party platforms."
711-
},
712-
new OpenAiRealtimeToolDto
713-
{
714-
Type = "function",
715-
Name = OpenAiToolConstants.HandleThirdPartyUnexpectedIssues,
716-
Description = "Handle undefined or unexpected issues with orders placed through third-party platforms."
717-
},
718-
new OpenAiRealtimeToolDto
719-
{
720-
Type = "function",
721-
Name = OpenAiToolConstants.HandlePromotionCalls,
722-
Description = "Handles calls not related to the restaurant related to advertising, promotions, insurance or product marketing."
723-
},
724-
new OpenAiRealtimeToolDto
725-
{
726-
Type = "function",
727-
Name = OpenAiToolConstants.CheckOrderStatus,
728-
Description = "Check the status of a customer's order, including whether it is prepared and ready for pickup or delivery."
729-
},
730-
new OpenAiRealtimeToolDto
731-
{
732-
Type = "function",
733-
Name = OpenAiToolConstants.RequestOrderDelivery,
734-
Description = "When customers request delivery of their orders"
735-
},
736-
new OpenAiRealtimeToolDto
737-
{
738-
Type = "function",
739-
Name = OpenAiToolConstants.Hangup,
740-
Description = "When the customer says goodbye or something similar, hang up the phone"
741-
},
742-
new OpenAiRealtimeToolDto
743-
{
744-
Type = "function",
745-
Name = OpenAiToolConstants.ConfirmOrder,
746-
Description = "When the customer says that's enough, or clearly says he wants to place an order, the rest are not the final order, but just recording the order.",
747-
Parameters = new OpenAiRealtimeToolParametersDto
748-
{
749-
Type = "object",
750-
Properties = new
751-
{
752-
order_items = new
753-
{
754-
type = "array",
755-
description = "The current complete order after the guest has modified the order",
756-
items = new
757-
{
758-
type = "object",
759-
properties = new
760-
{
761-
item_name = new
762-
{
763-
type = "string",
764-
description = "Name of the item ordered"
765-
},
766-
quantity = new
767-
{
768-
type = "number",
769-
description = "New quantity for the item"
770-
},
771-
price = new
772-
{
773-
type = "string",
774-
description = "The price of the item multiplied by the quantity"
775-
},
776-
notes = new
777-
{
778-
type = "string",
779-
description = "Additional notes or specifications for the item"
780-
},
781-
specification = new
782-
{
783-
type = "string",
784-
description = "Specified item size, such as large, medium, and small"
785-
}
786-
}
787-
}
788-
},
789-
total_price = new
790-
{
791-
type = "number",
792-
description = "The total price of the customer order",
793-
}
794-
}
795-
}
796-
},
797-
new OpenAiRealtimeToolDto
798-
{
799-
Type = "function",
800-
Name = OpenAiToolConstants.ConfirmCustomerInformation,
801-
Description = "When the customer confirms the order using their name and phone number, record this information.",
802-
Parameters = new OpenAiRealtimeToolParametersDto
803-
{
804-
Type = "object",
805-
Properties = new
806-
{
807-
customer_name = new
808-
{
809-
type = "string",
810-
description = "Name of the customer"
811-
},
812-
customer_phone = new
813-
{
814-
type = "string",
815-
description = "The phone number the customer used to place the pickup order"
816-
}
817-
}
818-
}
819-
}
820-
}
705+
tools = tools
821706
}
822707
};
823708

824709
await SendToWebSocketAsync(openAiWebSocket, sessionUpdate);
825710
}
711+
712+
private async Task<IEnumerable<OpenAiRealtimeToolDto>> InitialSessionToolsAsync(Domain.AISpeechAssistant.AiSpeechAssistant assistant, CancellationToken cancellationToken = default)
713+
{
714+
var functions = await _aiSpeechAssistantDataProvider.GetAiSpeechAssistantFunctionCallByAssistantIdAsync(assistant.Id, cancellationToken).ConfigureAwait(false);
715+
716+
return functions.Count == 0 ? [] : functions.Where(x => !string.IsNullOrWhiteSpace(x.Content)).Select(x => JsonConvert.DeserializeObject<OpenAiRealtimeToolDto>(x.Content));
717+
}
826718
}

‎src/SmartTalk.Core/Services/PhoneOrder/PhoneOrderService.OrderItem.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ public async Task<PlaceOrderAndModifyItemResponse> PlaceOrderAndModifyItemsAsync
6060
var request = new PlaceOrderToEasyPosRequestDto
6161
{
6262
Type = 1,
63-
IsTaxFree = true,
64-
Notes = string.Empty,
63+
IsTaxFree = false,
64+
Notes = record.Comments,
6565
OrderItems = orderItems.Select(x => new PhoneCallOrderItem
6666
{
6767
ProductId = x.ProductId ?? GetMenuItemByName(menuItems, x.FoodName)?.ProductId ?? 0,

‎src/SmartTalk.Core/Services/PhoneOrder/PhoneOrderService.Record.cs

+3-3
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ public async Task ReceivePhoneOrderRecordAsync(ReceivePhoneOrderRecordCommand co
5656
Log.Information("Phone order record information: {@recordInfo}", recordInfo);
5757

5858
if (recordInfo == null) return;
59-
if (await CheckOrderExistAsync(recordInfo.StartDate.AddHours(-8), cancellationToken).ConfigureAwait(false)) return;
59+
if (await CheckOrderExistAsync(recordInfo.StartDate, cancellationToken).ConfigureAwait(false)) return;
6060

6161
var transcription = await _speechToTextService.SpeechToTextAsync(
6262
command.RecordContent, fileType: TranscriptionFileType.Wav, responseFormat: TranscriptionResponseFormat.Text, cancellationToken: cancellationToken).ConfigureAwait(false);
@@ -65,7 +65,7 @@ public async Task ReceivePhoneOrderRecordAsync(ReceivePhoneOrderRecordCommand co
6565

6666
Log.Information("Phone order record transcription detected language: {@detectionLanguage}", detection.Language);
6767

68-
var record = new PhoneOrderRecord { SessionId = Guid.NewGuid().ToString(), AgentId = recordInfo.Agent.Id, TranscriptionText = transcription, Language = SelectLanguageEnum(detection.Language), CreatedDate = recordInfo.StartDate.AddHours(-8), Status = PhoneOrderRecordStatus.Recieved };
68+
var record = new PhoneOrderRecord { SessionId = Guid.NewGuid().ToString(), AgentId = recordInfo.Agent.Id, TranscriptionText = transcription, Language = SelectLanguageEnum(detection.Language), CreatedDate = recordInfo.StartDate, Status = PhoneOrderRecordStatus.Recieved };
6969

7070
if (await CheckPhoneOrderRecordDurationAsync(command.RecordContent, cancellationToken).ConfigureAwait(false))
7171
{
@@ -445,7 +445,7 @@ private DateTimeOffset ExtractPhoneOrderStartDateFromRecordName(string recordNam
445445

446446
if (match.Success) time = match.Groups[1].Value;
447447

448-
return DateTimeOffset.FromUnixTimeSeconds(long.Parse(time));
448+
return TimeZoneInfo.ConvertTime(DateTimeOffset.FromUnixTimeSeconds(long.Parse(time)), TimeZoneInfo.FindSystemTimeZoneById("America/Los_Angeles"));
449449
}
450450

451451
private async Task UpdatePhoneOrderRecordSpecificFieldsAsync(int recordId, int modifiedBy, string tips, string lastModifiedByName, CancellationToken cancellationToken)

0 commit comments

Comments
 (0)