Add chat API (#436)

* Add chat APIs - wip

* generated protobuf

* updates

* fix types and build

* Add conversion traits and update API

* fix unwrapping

* Populate timestamps and add edit support

* fix default values

* generated protobuf

* fix match

* remove explicit resolver

* move conversion

* error handling

* no imports

---------

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

Author: lukasIO

Cargo.lock

@@ -96,6 +96,9 @@ message FfiRequest {
     NewSoxResamplerRequest new_sox_resampler = 33;
     PushSoxResamplerRequest push_sox_resampler = 34;
     FlushSoxResamplerRequest flush_sox_resampler = 35;
+
+    SendChatMessageRequest send_chat_message = 36;
+    EditChatMessageRequest edit_chat_message = 37;
   }
 }
 
@@ -144,6 +147,8 @@ message FfiResponse {
     NewSoxResamplerResponse new_sox_resampler = 33;
     PushSoxResamplerResponse push_sox_resampler = 34;
     FlushSoxResamplerResponse flush_sox_resampler = 35;
+
+    SendChatMessageResponse send_chat_message = 36;
   }
 }
 
@@ -172,6 +177,7 @@ message FfiEvent {
     GetSessionStatsCallback get_session_stats = 19;
     Panic panic = 20;
     PublishSipDtmfCallback publish_sip_dtmf = 21;
+    SendChatMessageCallback chat_message = 22;
   }
 }
 

livekit-ffi/protocol/ffi.proto

@@ -144,6 +144,28 @@ message SetLocalMetadataCallback {
   optional string error = 2;
 }
 
+message SendChatMessageRequest {
+  uint64 local_participant_handle = 1;
+  string message = 2;
+  repeated string destination_identities = 3;
+  optional string sender_identity = 4;
+}
+message EditChatMessageRequest {
+  uint64 local_participant_handle = 1;
+  string edit_text = 2;
+  ChatMessage original_message = 3;
+  repeated string destination_identities = 4;
+  optional string sender_identity = 5;
+}
+message SendChatMessageResponse {
+  uint64 async_id = 1;
+}
+message SendChatMessageCallback {
+  uint64 async_id = 1;
+  optional string error = 2;
+  optional ChatMessage chat_message = 3;
+}
+
 // Change the local participant's attributes
 message SetLocalAttributesRequest {
   uint64 local_participant_handle = 1;
@@ -343,6 +365,7 @@ message RoomEvent {
     RoomEOS eos = 26; // The stream of room events has ended
     DataPacketReceived data_packet_received = 27;
     TranscriptionReceived transcription_received = 28;
+    ChatMessageReceived chat_message = 29;
   }
 }
 
@@ -457,6 +480,20 @@ message UserPacket {
   optional string topic = 2;
 }
 
+message ChatMessage {
+  string id = 1;
+  int64 timestamp = 2;
+  string message = 3;
+  optional int64 edit_timestamp = 4;
+  optional bool deleted = 5;
+  optional bool generated = 6;
+}
+
+message ChatMessageReceived {
+  ChatMessage message = 1;
+  string participant_identity = 2;
+}
+
 message SipDTMF {
   uint32 code = 1;
   optional string digit = 2;

livekit-ffi/protocol/room.proto

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::{proto, server::room::FfiRoom};
 use livekit::{
     e2ee::{
         key_provider::{KeyProvider, KeyProviderOptions},
@@ -25,8 +26,6 @@ use livekit::{
     },
 };
 
-use crate::{proto, server::room::FfiRoom};
-
 impl From<EncryptionState> for proto::EncryptionState {
     fn from(value: EncryptionState) -> Self {
         match value {
@@ -241,3 +240,29 @@ impl From<&FfiRoom> for proto::RoomInfo {
         }
     }
 }
+
+impl From<proto::ChatMessage> for livekit::ChatMessage {
+    fn from(proto_msg: proto::ChatMessage) -> Self {
+        livekit::ChatMessage {
+            id: proto_msg.id,
+            message: proto_msg.message,
+            timestamp: proto_msg.timestamp,
+            edit_timestamp: proto_msg.edit_timestamp,
+            deleted: proto_msg.deleted,
+            generated: proto_msg.generated,
+        }
+    }
+}
+
+impl From<livekit::ChatMessage> for proto::ChatMessage {
+    fn from(msg: livekit::ChatMessage) -> Self {
+        proto::ChatMessage {
+            id: msg.id,
+            message: msg.message,
+            timestamp: msg.timestamp,
+            edit_timestamp: msg.edit_timestamp,
+            deleted: msg.deleted.into(),
+            generated: msg.generated.into(),
+        }
+    }
+}

livekit-ffi/src/conversion/room.rs

@@ -2210,6 +2210,48 @@ pub struct SetLocalMetadataCallback {
     #[prost(string, optional, tag="2")]
     pub error: ::core::option::Option<::prost::alloc::string::String>,
 }
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct SendChatMessageRequest {
+    #[prost(uint64, tag="1")]
+    pub local_participant_handle: u64,
+    #[prost(string, tag="2")]
+    pub message: ::prost::alloc::string::String,
+    #[prost(string, repeated, tag="3")]
+    pub destination_identities: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+    #[prost(string, optional, tag="4")]
+    pub sender_identity: ::core::option::Option<::prost::alloc::string::String>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct EditChatMessageRequest {
+    #[prost(uint64, tag="1")]
+    pub local_participant_handle: u64,
+    #[prost(string, tag="2")]
+    pub edit_text: ::prost::alloc::string::String,
+    #[prost(message, optional, tag="3")]
+    pub original_message: ::core::option::Option<ChatMessage>,
+    #[prost(string, repeated, tag="4")]
+    pub destination_identities: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+    #[prost(string, optional, tag="5")]
+    pub sender_identity: ::core::option::Option<::prost::alloc::string::String>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct SendChatMessageResponse {
+    #[prost(uint64, tag="1")]
+    pub async_id: u64,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct SendChatMessageCallback {
+    #[prost(uint64, tag="1")]
+    pub async_id: u64,
+    #[prost(string, optional, tag="2")]
+    pub error: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(message, optional, tag="3")]
+    pub chat_message: ::core::option::Option<ChatMessage>,
+}
 /// Change the local participant's attributes
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -2407,7 +2449,7 @@ pub struct OwnedBuffer {
 pub struct RoomEvent {
     #[prost(uint64, tag="1")]
     pub room_handle: u64,
-    #[prost(oneof="room_event::Message", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28")]
+    #[prost(oneof="room_event::Message", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29")]
     pub message: ::core::option::Option<room_event::Message>,
 }
 /// Nested message and enum types in `RoomEvent`.
@@ -2471,6 +2513,8 @@ pub mod room_event {
         DataPacketReceived(super::DataPacketReceived),
         #[prost(message, tag="28")]
         TranscriptionReceived(super::TranscriptionReceived),
+        #[prost(message, tag="29")]
+        ChatMessage(super::ChatMessageReceived),
     }
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
@@ -2655,6 +2699,30 @@ pub struct UserPacket {
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ChatMessage {
+    #[prost(string, tag="1")]
+    pub id: ::prost::alloc::string::String,
+    #[prost(int64, tag="2")]
+    pub timestamp: i64,
+    #[prost(string, tag="3")]
+    pub message: ::prost::alloc::string::String,
+    #[prost(int64, optional, tag="4")]
+    pub edit_timestamp: ::core::option::Option<i64>,
+    #[prost(bool, optional, tag="5")]
+    pub deleted: ::core::option::Option<bool>,
+    #[prost(bool, optional, tag="6")]
+    pub generated: ::core::option::Option<bool>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ChatMessageReceived {
+    #[prost(message, optional, tag="1")]
+    pub message: ::core::option::Option<ChatMessage>,
+    #[prost(string, tag="2")]
+    pub participant_identity: ::prost::alloc::string::String,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SipDtmf {
     #[prost(uint32, tag="1")]
     pub code: u32,
@@ -3443,7 +3511,7 @@ impl AudioSourceType {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct FfiRequest {
-    #[prost(oneof="ffi_request::Message", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35")]
+    #[prost(oneof="ffi_request::Message", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37")]
     pub message: ::core::option::Option<ffi_request::Message>,
 }
 /// Nested message and enum types in `FfiRequest`.
@@ -3523,13 +3591,17 @@ pub mod ffi_request {
         PushSoxResampler(super::PushSoxResamplerRequest),
         #[prost(message, tag="35")]
         FlushSoxResampler(super::FlushSoxResamplerRequest),
+        #[prost(message, tag="36")]
+        SendChatMessage(super::SendChatMessageRequest),
+        #[prost(message, tag="37")]
+        EditChatMessage(super::EditChatMessageRequest),
     }
 }
 /// This is the output of livekit_ffi_request function.
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct FfiResponse {
-    #[prost(oneof="ffi_response::Message", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35")]
+    #[prost(oneof="ffi_response::Message", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36")]
     pub message: ::core::option::Option<ffi_response::Message>,
 }
 /// Nested message and enum types in `FfiResponse`.
@@ -3609,6 +3681,8 @@ pub mod ffi_response {
         PushSoxResampler(super::PushSoxResamplerResponse),
         #[prost(message, tag="35")]
         FlushSoxResampler(super::FlushSoxResamplerResponse),
+        #[prost(message, tag="36")]
+        SendChatMessage(super::SendChatMessageResponse),
     }
 }
 /// To minimize complexity, participant events are not included in the protocol.
@@ -3617,7 +3691,7 @@ pub mod ffi_response {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct FfiEvent {
-    #[prost(oneof="ffi_event::Message", tags="1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21")]
+    #[prost(oneof="ffi_event::Message", tags="1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22")]
     pub message: ::core::option::Option<ffi_event::Message>,
 }
 /// Nested message and enum types in `FfiEvent`.
@@ -3665,6 +3739,8 @@ pub mod ffi_event {
         Panic(super::Panic),
         #[prost(message, tag="21")]
         PublishSipDtmf(super::PublishSipDtmfCallback),
+        #[prost(message, tag="22")]
+        ChatMessage(super::SendChatMessageCallback),
     }
 }
 /// Stop all rooms synchronously (Do we need async here?).