From adf8da0a33163a13d260c530f67f681eb4257467 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 25 Sep 2024 14:31:25 +0200 Subject: [PATCH 1/7] Add chat API support --- packages/livekit-rtc/rust-sdks | 2 +- packages/livekit-rtc/src/participant.ts | 29 +++ packages/livekit-rtc/src/proto/ffi_pb.ts | 23 +- packages/livekit-rtc/src/proto/room_pb.ts | 258 ++++++++++++++++++++++ 4 files changed, 310 insertions(+), 2 deletions(-) diff --git a/packages/livekit-rtc/rust-sdks b/packages/livekit-rtc/rust-sdks index 1b87c1af..96149632 160000 --- a/packages/livekit-rtc/rust-sdks +++ b/packages/livekit-rtc/rust-sdks @@ -1 +1 @@ -Subproject commit 1b87c1aff6fb84284ab15242843ff7222ed7dbf4 +Subproject commit 961496320e235032f601092b3930553e0c4a7aa6 diff --git a/packages/livekit-rtc/src/participant.ts b/packages/livekit-rtc/src/participant.ts index 63e24266..f4bfedd6 100644 --- a/packages/livekit-rtc/src/participant.ts +++ b/packages/livekit-rtc/src/participant.ts @@ -12,6 +12,8 @@ import type { PublishTrackResponse, PublishTranscriptionCallback, PublishTranscriptionResponse, + SendChatMessageCallback, + SendChatMessageResponse, SetLocalAttributesCallback, SetLocalAttributesResponse, SetLocalMetadataCallback, @@ -28,6 +30,7 @@ import { PublishSipDtmfRequest, PublishTrackRequest, PublishTranscriptionRequest, + SendChatMessageRequest, SetLocalAttributesRequest, SetLocalMetadataRequest, SetLocalNameRequest, @@ -185,6 +188,32 @@ export class LocalParticipant extends Participant { }); } + async sendChatMessage( + text: string, + destinationIdentities?: Array, + senderIdentity?: string, + ) { + const req = new SendChatMessageRequest({ + localParticipantHandle: this.ffi_handle.handle, + message: text, + destinationIdentities, + senderIdentity, + }); + + const res = FfiClient.instance.request({ + message: { case: 'sendChatMessage', value: req }, + }); + + const cb = await FfiClient.instance.waitFor((ev) => { + return ev.message.case == 'chatMessage' && ev.message.value.asyncId == res.asyncId; + }); + + if (cb.error) { + throw new Error(cb.error); + } + return cb.chatMessage!; + } + async updateName(name: string) { const req = new SetLocalNameRequest({ localParticipantHandle: this.ffi_handle.handle, diff --git a/packages/livekit-rtc/src/proto/ffi_pb.ts b/packages/livekit-rtc/src/proto/ffi_pb.ts index e74b4809..57cae8d5 100644 --- a/packages/livekit-rtc/src/proto/ffi_pb.ts +++ b/packages/livekit-rtc/src/proto/ffi_pb.ts @@ -19,7 +19,7 @@ import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialMessage, PlainMessage } from "@bufbuild/protobuf"; import { Message, proto3, protoInt64 } from "@bufbuild/protobuf"; -import { ConnectCallback, ConnectRequest, ConnectResponse, DisconnectCallback, DisconnectRequest, DisconnectResponse, GetSessionStatsCallback, GetSessionStatsRequest, GetSessionStatsResponse, PublishDataCallback, PublishDataRequest, PublishDataResponse, PublishSipDtmfCallback, PublishSipDtmfRequest, PublishSipDtmfResponse, PublishTrackCallback, PublishTrackRequest, PublishTrackResponse, PublishTranscriptionCallback, PublishTranscriptionRequest, PublishTranscriptionResponse, RoomEvent, SetLocalAttributesCallback, SetLocalAttributesRequest, SetLocalAttributesResponse, SetLocalMetadataCallback, SetLocalMetadataRequest, SetLocalMetadataResponse, SetLocalNameCallback, SetLocalNameRequest, SetLocalNameResponse, SetSubscribedRequest, SetSubscribedResponse, UnpublishTrackCallback, UnpublishTrackRequest, UnpublishTrackResponse } from "./room_pb.js"; +import { ConnectCallback, ConnectRequest, ConnectResponse, DisconnectCallback, DisconnectRequest, DisconnectResponse, GetSessionStatsCallback, GetSessionStatsRequest, GetSessionStatsResponse, PublishDataCallback, PublishDataRequest, PublishDataResponse, PublishSipDtmfCallback, PublishSipDtmfRequest, PublishSipDtmfResponse, PublishTrackCallback, PublishTrackRequest, PublishTrackResponse, PublishTranscriptionCallback, PublishTranscriptionRequest, PublishTranscriptionResponse, RoomEvent, SendChatMessageCallback, SendChatMessageRequest, SendChatMessageResponse, SetLocalAttributesCallback, SetLocalAttributesRequest, SetLocalAttributesResponse, SetLocalMetadataCallback, SetLocalMetadataRequest, SetLocalMetadataResponse, SetLocalNameCallback, SetLocalNameRequest, SetLocalNameResponse, SetSubscribedRequest, SetSubscribedResponse, UnpublishTrackCallback, UnpublishTrackRequest, UnpublishTrackResponse } from "./room_pb.js"; import { CreateAudioTrackRequest, CreateAudioTrackResponse, CreateVideoTrackRequest, CreateVideoTrackResponse, EnableRemoteTrackRequest, EnableRemoteTrackResponse, GetStatsCallback, GetStatsRequest, GetStatsResponse, LocalTrackMuteRequest, LocalTrackMuteResponse, TrackEvent } from "./track_pb.js"; import { CaptureVideoFrameRequest, CaptureVideoFrameResponse, NewVideoSourceRequest, NewVideoSourceResponse, NewVideoStreamRequest, NewVideoStreamResponse, VideoConvertRequest, VideoConvertResponse, VideoStreamEvent, VideoStreamFromParticipantRequest, VideoStreamFromParticipantResponse } from "./video_frame_pb.js"; import { AudioStreamEvent, AudioStreamFromParticipantRequest, AudioStreamFromParticipantResponse, CaptureAudioFrameCallback, CaptureAudioFrameRequest, CaptureAudioFrameResponse, ClearAudioBufferRequest, ClearAudioBufferResponse, NewAudioResamplerRequest, NewAudioResamplerResponse, NewAudioSourceRequest, NewAudioSourceResponse, NewAudioStreamRequest, NewAudioStreamResponse, RemixAndResampleRequest, RemixAndResampleResponse } from "./audio_frame_pb.js"; @@ -153,6 +153,12 @@ export class FfiRequest extends Message { */ value: PublishSipDtmfRequest; case: "publishSipDtmf"; + } | { + /** + * @generated from field: livekit.proto.SendChatMessageRequest send_chat_message = 33; + */ + value: SendChatMessageRequest; + case: "sendChatMessage"; } | { /** * Track @@ -290,6 +296,7 @@ export class FfiRequest extends Message { { no: 12, name: "get_session_stats", kind: "message", T: GetSessionStatsRequest, oneof: "message" }, { no: 13, name: "publish_transcription", kind: "message", T: PublishTranscriptionRequest, oneof: "message" }, { no: 14, name: "publish_sip_dtmf", kind: "message", T: PublishSipDtmfRequest, oneof: "message" }, + { no: 33, name: "send_chat_message", kind: "message", T: SendChatMessageRequest, oneof: "message" }, { no: 15, name: "create_video_track", kind: "message", T: CreateVideoTrackRequest, oneof: "message" }, { no: 16, name: "create_audio_track", kind: "message", T: CreateAudioTrackRequest, oneof: "message" }, { no: 17, name: "local_track_mute", kind: "message", T: LocalTrackMuteRequest, oneof: "message" }, @@ -416,6 +423,12 @@ export class FfiResponse extends Message { */ value: PublishSipDtmfResponse; case: "publishSipDtmf"; + } | { + /** + * @generated from field: livekit.proto.SendChatMessageResponse send_chat_message = 33; + */ + value: SendChatMessageResponse; + case: "sendChatMessage"; } | { /** * Track @@ -553,6 +566,7 @@ export class FfiResponse extends Message { { no: 12, name: "get_session_stats", kind: "message", T: GetSessionStatsResponse, oneof: "message" }, { no: 13, name: "publish_transcription", kind: "message", T: PublishTranscriptionResponse, oneof: "message" }, { no: 14, name: "publish_sip_dtmf", kind: "message", T: PublishSipDtmfResponse, oneof: "message" }, + { no: 33, name: "send_chat_message", kind: "message", T: SendChatMessageResponse, oneof: "message" }, { no: 15, name: "create_video_track", kind: "message", T: CreateVideoTrackResponse, oneof: "message" }, { no: 16, name: "create_audio_track", kind: "message", T: CreateAudioTrackResponse, oneof: "message" }, { no: 17, name: "local_track_mute", kind: "message", T: LocalTrackMuteResponse, oneof: "message" }, @@ -721,6 +735,12 @@ export class FfiEvent extends Message { */ value: PublishSipDtmfCallback; case: "publishSipDtmf"; + } | { + /** + * @generated from field: livekit.proto.SendChatMessageCallback chat_message = 22; + */ + value: SendChatMessageCallback; + case: "chatMessage"; } | { case: undefined; value?: undefined } = { case: undefined }; constructor(data?: PartialMessage) { @@ -751,6 +771,7 @@ export class FfiEvent extends Message { { no: 19, name: "get_session_stats", kind: "message", T: GetSessionStatsCallback, oneof: "message" }, { no: 20, name: "panic", kind: "message", T: Panic, oneof: "message" }, { no: 21, name: "publish_sip_dtmf", kind: "message", T: PublishSipDtmfCallback, oneof: "message" }, + { no: 22, name: "chat_message", kind: "message", T: SendChatMessageCallback, oneof: "message" }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): FfiEvent { diff --git a/packages/livekit-rtc/src/proto/room_pb.ts b/packages/livekit-rtc/src/proto/room_pb.ts index fefc54b7..5e054bab 100644 --- a/packages/livekit-rtc/src/proto/room_pb.ts +++ b/packages/livekit-rtc/src/proto/room_pb.ts @@ -1375,6 +1375,147 @@ export class SetLocalMetadataCallback extends Message } } +/** + * @generated from message livekit.proto.SendChatMessageRequest + */ +export class SendChatMessageRequest extends Message { + /** + * @generated from field: uint64 local_participant_handle = 1; + */ + localParticipantHandle = protoInt64.zero; + + /** + * @generated from field: string message = 2; + */ + message = ""; + + /** + * @generated from field: repeated string destination_identities = 3; + */ + destinationIdentities: string[] = []; + + /** + * @generated from field: optional string sender_identity = 4; + */ + senderIdentity?: string; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "livekit.proto.SendChatMessageRequest"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "local_participant_handle", kind: "scalar", T: 4 /* ScalarType.UINT64 */ }, + { no: 2, name: "message", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 3, name: "destination_identities", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true }, + { no: 4, name: "sender_identity", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): SendChatMessageRequest { + return new SendChatMessageRequest().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): SendChatMessageRequest { + return new SendChatMessageRequest().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): SendChatMessageRequest { + return new SendChatMessageRequest().fromJsonString(jsonString, options); + } + + static equals(a: SendChatMessageRequest | PlainMessage | undefined, b: SendChatMessageRequest | PlainMessage | undefined): boolean { + return proto3.util.equals(SendChatMessageRequest, a, b); + } +} + +/** + * @generated from message livekit.proto.SendChatMessageResponse + */ +export class SendChatMessageResponse extends Message { + /** + * @generated from field: uint64 async_id = 1; + */ + asyncId = protoInt64.zero; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "livekit.proto.SendChatMessageResponse"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "async_id", kind: "scalar", T: 4 /* ScalarType.UINT64 */ }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): SendChatMessageResponse { + return new SendChatMessageResponse().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): SendChatMessageResponse { + return new SendChatMessageResponse().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): SendChatMessageResponse { + return new SendChatMessageResponse().fromJsonString(jsonString, options); + } + + static equals(a: SendChatMessageResponse | PlainMessage | undefined, b: SendChatMessageResponse | PlainMessage | undefined): boolean { + return proto3.util.equals(SendChatMessageResponse, a, b); + } +} + +/** + * @generated from message livekit.proto.SendChatMessageCallback + */ +export class SendChatMessageCallback extends Message { + /** + * @generated from field: uint64 async_id = 1; + */ + asyncId = protoInt64.zero; + + /** + * @generated from field: optional string error = 2; + */ + error?: string; + + /** + * @generated from field: optional livekit.proto.ChatMessage chat_message = 3; + */ + chatMessage?: ChatMessage; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "livekit.proto.SendChatMessageCallback"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "async_id", kind: "scalar", T: 4 /* ScalarType.UINT64 */ }, + { no: 2, name: "error", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + { no: 3, name: "chat_message", kind: "message", T: ChatMessage, opt: true }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): SendChatMessageCallback { + return new SendChatMessageCallback().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): SendChatMessageCallback { + return new SendChatMessageCallback().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): SendChatMessageCallback { + return new SendChatMessageCallback().fromJsonString(jsonString, options); + } + + static equals(a: SendChatMessageCallback | PlainMessage | undefined, b: SendChatMessageCallback | PlainMessage | undefined): boolean { + return proto3.util.equals(SendChatMessageCallback, a, b); + } +} + /** * Change the local participant's attributes * @@ -2491,6 +2632,12 @@ export class RoomEvent extends Message { */ value: TranscriptionReceived; case: "transcriptionReceived"; + } | { + /** + * @generated from field: livekit.proto.ChatMessageReceived chat_message = 29; + */ + value: ChatMessageReceived; + case: "chatMessage"; } | { case: undefined; value?: undefined } = { case: undefined }; constructor(data?: PartialMessage) { @@ -2529,6 +2676,7 @@ export class RoomEvent extends Message { { no: 26, name: "eos", kind: "message", T: RoomEOS, oneof: "message" }, { no: 27, name: "data_packet_received", kind: "message", T: DataPacketReceived, oneof: "message" }, { no: 28, name: "transcription_received", kind: "message", T: TranscriptionReceived, oneof: "message" }, + { no: 29, name: "chat_message", kind: "message", T: ChatMessageReceived, oneof: "message" }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): RoomEvent { @@ -3517,6 +3665,116 @@ export class UserPacket extends Message { } } +/** + * @generated from message livekit.proto.ChatMessage + */ +export class ChatMessage extends Message { + /** + * @generated from field: string id = 1; + */ + id = ""; + + /** + * @generated from field: int64 timestamp = 2; + */ + timestamp = protoInt64.zero; + + /** + * @generated from field: string message = 3; + */ + message = ""; + + /** + * @generated from field: optional int64 edit_timestamp = 4; + */ + editTimestamp?: bigint; + + /** + * @generated from field: optional bool deleted = 5; + */ + deleted?: boolean; + + /** + * @generated from field: optional bool generated = 6; + */ + generated?: boolean; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "livekit.proto.ChatMessage"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 2, name: "timestamp", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 3, name: "message", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 4, name: "edit_timestamp", kind: "scalar", T: 3 /* ScalarType.INT64 */, opt: true }, + { no: 5, name: "deleted", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, + { no: 6, name: "generated", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): ChatMessage { + return new ChatMessage().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): ChatMessage { + return new ChatMessage().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): ChatMessage { + return new ChatMessage().fromJsonString(jsonString, options); + } + + static equals(a: ChatMessage | PlainMessage | undefined, b: ChatMessage | PlainMessage | undefined): boolean { + return proto3.util.equals(ChatMessage, a, b); + } +} + +/** + * @generated from message livekit.proto.ChatMessageReceived + */ +export class ChatMessageReceived extends Message { + /** + * @generated from field: livekit.proto.ChatMessage message = 1; + */ + message?: ChatMessage; + + /** + * @generated from field: string participant_identity = 2; + */ + participantIdentity = ""; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "livekit.proto.ChatMessageReceived"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "message", kind: "message", T: ChatMessage }, + { no: 2, name: "participant_identity", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): ChatMessageReceived { + return new ChatMessageReceived().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): ChatMessageReceived { + return new ChatMessageReceived().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): ChatMessageReceived { + return new ChatMessageReceived().fromJsonString(jsonString, options); + } + + static equals(a: ChatMessageReceived | PlainMessage | undefined, b: ChatMessageReceived | PlainMessage | undefined): boolean { + return proto3.util.equals(ChatMessageReceived, a, b); + } +} + /** * @generated from message livekit.proto.SipDTMF */ From e83ea7821949fb5d194a1dd681be6c2cd750414b Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 25 Sep 2024 14:45:04 +0200 Subject: [PATCH 2/7] Add receive support --- packages/livekit-rtc/src/index.ts | 1 + packages/livekit-rtc/src/room.ts | 15 ++++++++++++++- packages/livekit-rtc/src/types.ts | 7 +++++++ 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 packages/livekit-rtc/src/types.ts diff --git a/packages/livekit-rtc/src/index.ts b/packages/livekit-rtc/src/index.ts index b1973876..84175a9f 100644 --- a/packages/livekit-rtc/src/index.ts +++ b/packages/livekit-rtc/src/index.ts @@ -42,3 +42,4 @@ export { StreamState, TrackKind, TrackSource } from './proto/track_pb.js'; export { VideoBufferType, VideoRotation } from './proto/video_frame_pb.js'; export { ParticipantKind } from './proto/participant_pb.js'; export { dispose } from './ffi_client.js'; +export type { ChatMessage } from './types.js'; diff --git a/packages/livekit-rtc/src/room.ts b/packages/livekit-rtc/src/room.ts index fa2ae9f2..369ddaab 100644 --- a/packages/livekit-rtc/src/room.ts +++ b/packages/livekit-rtc/src/room.ts @@ -32,6 +32,7 @@ import type { LocalTrack, RemoteTrack } from './track.js'; import { RemoteAudioTrack, RemoteVideoTrack } from './track.js'; import type { LocalTrackPublication, TrackPublication } from './track_publication.js'; import { RemoteTrackPublication } from './track_publication.js'; +import type { ChatMessage } from './types.js'; export interface RtcConfiguration { iceTransportType: IceTransportType; @@ -262,6 +263,17 @@ export class Room extends (EventEmitter as new () => TypedEmitter } else if (ev.case == 'connectionQualityChanged') { const participant = this.retrieveParticipantByIdentity(ev.value.participantIdentity); this.emit(RoomEvent.ConnectionQualityChanged, ev.value.quality, participant); + } else if (ev.case == 'chatMessage') { + const participant = this.retrieveParticipantByIdentity(ev.value.participantIdentity); + const { id, message: messageText, timestamp, editTimestamp, generated } = ev.value.message; + const message: ChatMessage = { + id, + message: messageText, + timestamp: Number(timestamp), + editTimestamp: Number(editTimestamp), + generated, + }; + this.emit(RoomEvent.ChatMessage, message, participant); } else if (ev.case == 'dataPacketReceived') { // Can be undefined if the data is sent from a Server SDK const participant = this.remoteParticipants.get(ev.value.participantIdentity); @@ -285,7 +297,6 @@ export class Room extends (EventEmitter as new () => TypedEmitter const { code, digit } = dataPacket.value; this.emit(RoomEvent.DtmfReceived, code, digit, participant); break; - default: break; } @@ -376,6 +387,7 @@ export type RoomCallbacks = { kind?: DataPacketKind, topic?: string, ) => void; + chatMessage: (message: ChatMessage, participant?: Participant) => void; dtmfReceived: (code: number, digit: string, participant: RemoteParticipant) => void; encryptionError: (error: Error) => void; connectionStateChanged: (state: ConnectionState) => void; @@ -405,6 +417,7 @@ export enum RoomEvent { ParticipantAttributesChanged = 'participantAttributesChanged', ConnectionQualityChanged = 'connectionQualityChanged', DataReceived = 'dataReceived', + ChatMessage = 'chatMessage', DtmfReceived = 'dtmfReceived', EncryptionError = 'encryptionError', ConnectionStateChanged = 'connectionStateChanged', diff --git a/packages/livekit-rtc/src/types.ts b/packages/livekit-rtc/src/types.ts new file mode 100644 index 00000000..b120ad46 --- /dev/null +++ b/packages/livekit-rtc/src/types.ts @@ -0,0 +1,7 @@ +export interface ChatMessage { + id: string; + timestamp: number; + message: string; + editTimestamp?: number; + generated?: boolean; +} From 4c84ea9384587edbaafaab64b00d77129ad0d5f3 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 25 Sep 2024 15:18:39 +0200 Subject: [PATCH 3/7] Add edit message support --- packages/livekit-rtc/rust-sdks | 2 +- packages/livekit-rtc/src/participant.ts | 30 +++++++++++ packages/livekit-rtc/src/proto/ffi_pb.ts | 9 +++- packages/livekit-rtc/src/proto/room_pb.ts | 61 +++++++++++++++++++++++ 4 files changed, 100 insertions(+), 2 deletions(-) diff --git a/packages/livekit-rtc/rust-sdks b/packages/livekit-rtc/rust-sdks index 96149632..9363544e 160000 --- a/packages/livekit-rtc/rust-sdks +++ b/packages/livekit-rtc/rust-sdks @@ -1 +1 @@ -Subproject commit 961496320e235032f601092b3930553e0c4a7aa6 +Subproject commit 9363544e9b5a45690c9cf8b1b3f291aa7a151cca diff --git a/packages/livekit-rtc/src/participant.ts b/packages/livekit-rtc/src/participant.ts index f4bfedd6..50c6c21b 100644 --- a/packages/livekit-rtc/src/participant.ts +++ b/packages/livekit-rtc/src/participant.ts @@ -4,6 +4,7 @@ import { FfiClient, FfiHandle } from './ffi_client.js'; import type { OwnedParticipant, ParticipantInfo, ParticipantKind } from './proto/participant_pb.js'; import type { + ChatMessage, PublishDataCallback, PublishDataResponse, PublishSipDtmfCallback, @@ -25,6 +26,7 @@ import type { UnpublishTrackResponse, } from './proto/room_pb.js'; import { + EditChatMessageRequest, TranscriptionSegment as ProtoTranscriptionSegment, PublishDataRequest, PublishSipDtmfRequest, @@ -214,6 +216,34 @@ export class LocalParticipant extends Participant { return cb.chatMessage!; } + async editChatMessage( + editText: string, + originalMessage: ChatMessage, + destinationIdentities?: Array, + senderIdentity?: string, + ) { + const req = new EditChatMessageRequest({ + localParticipantHandle: this.ffi_handle.handle, + editText, + originalMessage, + destinationIdentities, + senderIdentity, + }); + + const res = FfiClient.instance.request({ + message: { case: 'sendChatMessage', value: req }, + }); + + const cb = await FfiClient.instance.waitFor((ev) => { + return ev.message.case == 'chatMessage' && ev.message.value.asyncId == res.asyncId; + }); + + if (cb.error) { + throw new Error(cb.error); + } + return cb.chatMessage!; + } + async updateName(name: string) { const req = new SetLocalNameRequest({ localParticipantHandle: this.ffi_handle.handle, diff --git a/packages/livekit-rtc/src/proto/ffi_pb.ts b/packages/livekit-rtc/src/proto/ffi_pb.ts index 57cae8d5..35c85f3e 100644 --- a/packages/livekit-rtc/src/proto/ffi_pb.ts +++ b/packages/livekit-rtc/src/proto/ffi_pb.ts @@ -19,7 +19,7 @@ import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialMessage, PlainMessage } from "@bufbuild/protobuf"; import { Message, proto3, protoInt64 } from "@bufbuild/protobuf"; -import { ConnectCallback, ConnectRequest, ConnectResponse, DisconnectCallback, DisconnectRequest, DisconnectResponse, GetSessionStatsCallback, GetSessionStatsRequest, GetSessionStatsResponse, PublishDataCallback, PublishDataRequest, PublishDataResponse, PublishSipDtmfCallback, PublishSipDtmfRequest, PublishSipDtmfResponse, PublishTrackCallback, PublishTrackRequest, PublishTrackResponse, PublishTranscriptionCallback, PublishTranscriptionRequest, PublishTranscriptionResponse, RoomEvent, SendChatMessageCallback, SendChatMessageRequest, SendChatMessageResponse, SetLocalAttributesCallback, SetLocalAttributesRequest, SetLocalAttributesResponse, SetLocalMetadataCallback, SetLocalMetadataRequest, SetLocalMetadataResponse, SetLocalNameCallback, SetLocalNameRequest, SetLocalNameResponse, SetSubscribedRequest, SetSubscribedResponse, UnpublishTrackCallback, UnpublishTrackRequest, UnpublishTrackResponse } from "./room_pb.js"; +import { ConnectCallback, ConnectRequest, ConnectResponse, DisconnectCallback, DisconnectRequest, DisconnectResponse, EditChatMessageRequest, GetSessionStatsCallback, GetSessionStatsRequest, GetSessionStatsResponse, PublishDataCallback, PublishDataRequest, PublishDataResponse, PublishSipDtmfCallback, PublishSipDtmfRequest, PublishSipDtmfResponse, PublishTrackCallback, PublishTrackRequest, PublishTrackResponse, PublishTranscriptionCallback, PublishTranscriptionRequest, PublishTranscriptionResponse, RoomEvent, SendChatMessageCallback, SendChatMessageRequest, SendChatMessageResponse, SetLocalAttributesCallback, SetLocalAttributesRequest, SetLocalAttributesResponse, SetLocalMetadataCallback, SetLocalMetadataRequest, SetLocalMetadataResponse, SetLocalNameCallback, SetLocalNameRequest, SetLocalNameResponse, SetSubscribedRequest, SetSubscribedResponse, UnpublishTrackCallback, UnpublishTrackRequest, UnpublishTrackResponse } from "./room_pb.js"; import { CreateAudioTrackRequest, CreateAudioTrackResponse, CreateVideoTrackRequest, CreateVideoTrackResponse, EnableRemoteTrackRequest, EnableRemoteTrackResponse, GetStatsCallback, GetStatsRequest, GetStatsResponse, LocalTrackMuteRequest, LocalTrackMuteResponse, TrackEvent } from "./track_pb.js"; import { CaptureVideoFrameRequest, CaptureVideoFrameResponse, NewVideoSourceRequest, NewVideoSourceResponse, NewVideoStreamRequest, NewVideoStreamResponse, VideoConvertRequest, VideoConvertResponse, VideoStreamEvent, VideoStreamFromParticipantRequest, VideoStreamFromParticipantResponse } from "./video_frame_pb.js"; import { AudioStreamEvent, AudioStreamFromParticipantRequest, AudioStreamFromParticipantResponse, CaptureAudioFrameCallback, CaptureAudioFrameRequest, CaptureAudioFrameResponse, ClearAudioBufferRequest, ClearAudioBufferResponse, NewAudioResamplerRequest, NewAudioResamplerResponse, NewAudioSourceRequest, NewAudioSourceResponse, NewAudioStreamRequest, NewAudioStreamResponse, RemixAndResampleRequest, RemixAndResampleResponse } from "./audio_frame_pb.js"; @@ -159,6 +159,12 @@ export class FfiRequest extends Message { */ value: SendChatMessageRequest; case: "sendChatMessage"; + } | { + /** + * @generated from field: livekit.proto.EditChatMessageRequest edit_chat_message = 34; + */ + value: EditChatMessageRequest; + case: "editChatMessage"; } | { /** * Track @@ -297,6 +303,7 @@ export class FfiRequest extends Message { { no: 13, name: "publish_transcription", kind: "message", T: PublishTranscriptionRequest, oneof: "message" }, { no: 14, name: "publish_sip_dtmf", kind: "message", T: PublishSipDtmfRequest, oneof: "message" }, { no: 33, name: "send_chat_message", kind: "message", T: SendChatMessageRequest, oneof: "message" }, + { no: 34, name: "edit_chat_message", kind: "message", T: EditChatMessageRequest, oneof: "message" }, { no: 15, name: "create_video_track", kind: "message", T: CreateVideoTrackRequest, oneof: "message" }, { no: 16, name: "create_audio_track", kind: "message", T: CreateAudioTrackRequest, oneof: "message" }, { no: 17, name: "local_track_mute", kind: "message", T: LocalTrackMuteRequest, oneof: "message" }, diff --git a/packages/livekit-rtc/src/proto/room_pb.ts b/packages/livekit-rtc/src/proto/room_pb.ts index 5e054bab..219f431c 100644 --- a/packages/livekit-rtc/src/proto/room_pb.ts +++ b/packages/livekit-rtc/src/proto/room_pb.ts @@ -1430,6 +1430,67 @@ export class SendChatMessageRequest extends Message { } } +/** + * @generated from message livekit.proto.EditChatMessageRequest + */ +export class EditChatMessageRequest extends Message { + /** + * @generated from field: uint64 local_participant_handle = 1; + */ + localParticipantHandle = protoInt64.zero; + + /** + * @generated from field: string edit_text = 2; + */ + editText = ""; + + /** + * @generated from field: livekit.proto.ChatMessage original_message = 3; + */ + originalMessage?: ChatMessage; + + /** + * @generated from field: repeated string destination_identities = 4; + */ + destinationIdentities: string[] = []; + + /** + * @generated from field: optional string sender_identity = 5; + */ + senderIdentity?: string; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "livekit.proto.EditChatMessageRequest"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "local_participant_handle", kind: "scalar", T: 4 /* ScalarType.UINT64 */ }, + { no: 2, name: "edit_text", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 3, name: "original_message", kind: "message", T: ChatMessage }, + { no: 4, name: "destination_identities", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true }, + { no: 5, name: "sender_identity", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): EditChatMessageRequest { + return new EditChatMessageRequest().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): EditChatMessageRequest { + return new EditChatMessageRequest().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): EditChatMessageRequest { + return new EditChatMessageRequest().fromJsonString(jsonString, options); + } + + static equals(a: EditChatMessageRequest | PlainMessage | undefined, b: EditChatMessageRequest | PlainMessage | undefined): boolean { + return proto3.util.equals(EditChatMessageRequest, a, b); + } +} + /** * @generated from message livekit.proto.SendChatMessageResponse */ From 76749ef95ff602a626ef630c3fdee11dac97f718 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 25 Sep 2024 15:33:37 +0200 Subject: [PATCH 4/7] Fix editing --- packages/livekit-rtc/rust-sdks | 2 +- packages/livekit-rtc/src/participant.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/livekit-rtc/rust-sdks b/packages/livekit-rtc/rust-sdks index 9363544e..078f6965 160000 --- a/packages/livekit-rtc/rust-sdks +++ b/packages/livekit-rtc/rust-sdks @@ -1 +1 @@ -Subproject commit 9363544e9b5a45690c9cf8b1b3f291aa7a151cca +Subproject commit 078f69658af8bee02184f7eb7888d5256df6be19 diff --git a/packages/livekit-rtc/src/participant.ts b/packages/livekit-rtc/src/participant.ts index 50c6c21b..6c519240 100644 --- a/packages/livekit-rtc/src/participant.ts +++ b/packages/livekit-rtc/src/participant.ts @@ -231,7 +231,7 @@ export class LocalParticipant extends Participant { }); const res = FfiClient.instance.request({ - message: { case: 'sendChatMessage', value: req }, + message: { case: 'editChatMessage', value: req }, }); const cb = await FfiClient.instance.waitFor((ev) => { From 23bf1d752678a0634b574d24923e50227f2c682f Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 25 Sep 2024 15:35:27 +0200 Subject: [PATCH 5/7] reuse --- packages/livekit-rtc/src/types.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/livekit-rtc/src/types.ts b/packages/livekit-rtc/src/types.ts index b120ad46..0f98f0b4 100644 --- a/packages/livekit-rtc/src/types.ts +++ b/packages/livekit-rtc/src/types.ts @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2024 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 export interface ChatMessage { id: string; timestamp: number; From 52a57f20d89318fa5e64bd43aa2cbcf1be9e79b5 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 7 Oct 2024 16:12:37 +0200 Subject: [PATCH 6/7] fix types --- packages/livekit-rtc/src/participant.ts | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/livekit-rtc/src/participant.ts b/packages/livekit-rtc/src/participant.ts index 6c519240..e5759074 100644 --- a/packages/livekit-rtc/src/participant.ts +++ b/packages/livekit-rtc/src/participant.ts @@ -3,8 +3,8 @@ // SPDX-License-Identifier: Apache-2.0 import { FfiClient, FfiHandle } from './ffi_client.js'; import type { OwnedParticipant, ParticipantInfo, ParticipantKind } from './proto/participant_pb.js'; -import type { - ChatMessage, +import { + ChatMessage as ChatMessageModel, PublishDataCallback, PublishDataResponse, PublishSipDtmfCallback, @@ -42,6 +42,7 @@ import type { LocalTrack } from './track.js'; import type { RemoteTrackPublication, TrackPublication } from './track_publication.js'; import { LocalTrackPublication } from './track_publication.js'; import type { Transcription } from './transcription.js'; +import { ChatMessage } from './types.js'; export abstract class Participant { /** @internal */ @@ -194,7 +195,7 @@ export class LocalParticipant extends Participant { text: string, destinationIdentities?: Array, senderIdentity?: string, - ) { + ): Promise { const req = new SendChatMessageRequest({ localParticipantHandle: this.ffi_handle.handle, message: text, @@ -213,7 +214,8 @@ export class LocalParticipant extends Participant { if (cb.error) { throw new Error(cb.error); } - return cb.chatMessage!; + const { id, timestamp, editTimestamp, message } = cb.chatMessage; + return { id, timestamp: Number(timestamp), editTimestamp: Number(editTimestamp), message }; } async editChatMessage( @@ -221,11 +223,15 @@ export class LocalParticipant extends Participant { originalMessage: ChatMessage, destinationIdentities?: Array, senderIdentity?: string, - ) { + ): Promise { const req = new EditChatMessageRequest({ localParticipantHandle: this.ffi_handle.handle, editText, - originalMessage, + originalMessage: new ChatMessageModel({ + ...originalMessage, + timestamp: BigInt(originalMessage.timestamp), + editTimestamp: BigInt(originalMessage.editTimestamp), + }), destinationIdentities, senderIdentity, }); @@ -241,7 +247,8 @@ export class LocalParticipant extends Participant { if (cb.error) { throw new Error(cb.error); } - return cb.chatMessage!; + const { id, timestamp, editTimestamp, message } = cb.chatMessage; + return { id, timestamp: Number(timestamp), editTimestamp: Number(editTimestamp), message }; } async updateName(name: string) { From 31830864f7e08337e1d895d749cc75391810f4c3 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 14 Oct 2024 13:57:12 +0200 Subject: [PATCH 7/7] Create funny-lamps-carry.md --- .changeset/funny-lamps-carry.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/funny-lamps-carry.md diff --git a/.changeset/funny-lamps-carry.md b/.changeset/funny-lamps-carry.md new file mode 100644 index 00000000..5f13d0e0 --- /dev/null +++ b/.changeset/funny-lamps-carry.md @@ -0,0 +1,5 @@ +--- +"@livekit/rtc-node": patch +--- + +Add chat API support