From a6305e1f31108ffa2ee4fc76bb84d0f14bc2a961 Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 1 Aug 2023 03:56:17 +0900 Subject: [PATCH 01/42] =?UTF-8?q?=E6=8E=A5=E7=B6=9A=E4=B8=AD=E3=81=AB?= =?UTF-8?q?=E3=82=AD=E3=83=A3=E3=83=97=E3=83=81=E3=83=A3=E3=83=A9=E3=82=92?= =?UTF-8?q?=E5=88=87=E3=82=8A=E6=9B=BF=E3=81=88=E3=82=8B=E6=A9=9F=E8=83=BD?= =?UTF-8?q?=E3=82=92=E5=AE=9F=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sora/Sora.cs | 94 ++++++++++++++++++++++----- proto/sora_conf_internal.proto | 16 +++-- src/sora.cpp | 115 +++++++++++++++++++++++---------- src/sora.h | 3 + src/unity.cpp | 7 ++ src/unity.h | 1 + src/unity_renderer.cpp | 19 ++++++ src/unity_renderer.h | 3 + 8 files changed, 203 insertions(+), 55 deletions(-) diff --git a/Sora/Sora.cs b/Sora/Sora.cs index c48b5c8..2cdfce0 100644 --- a/Sora/Sora.cs +++ b/Sora/Sora.cs @@ -83,6 +83,44 @@ public class Rule public List> Rules = new List>(); } + public class CameraConfig + { + public CapturerType CapturerType = Sora.CapturerType.DeviceCamera; + public UnityEngine.Camera UnityCamera = null; + public int UnityCameraRenderTargetDepthBuffer = 16; + public string VideoCapturerDevice = ""; + public int VideoWidth = 640; + public int VideoHeight = 480; + public int VideoFps = 30; + + public static CameraConfig FromUnityCamera(UnityEngine.Camera unityCamera, int unityCameraRenderTargetDepthBuffer, int videoWidth, int videoHeight, int videoFps) + { + return new CameraConfig() + { + CapturerType = Sora.CapturerType.UnityCamera, + UnityCamera = unityCamera, + UnityCameraRenderTargetDepthBuffer = unityCameraRenderTargetDepthBuffer, + VideoCapturerDevice = "", + VideoWidth = videoWidth, + VideoHeight = videoHeight, + VideoFps = videoFps, + }; + } + public static CameraConfig FromDeviceCamera(string videoCapturerDevice, int videoWidth, int videoHeight, int videoFps) + { + return new CameraConfig() + { + CapturerType = Sora.CapturerType.DeviceCamera, + UnityCamera = null, + UnityCameraRenderTargetDepthBuffer = 16, + VideoCapturerDevice = videoCapturerDevice, + VideoWidth = videoWidth, + VideoHeight = videoHeight, + VideoFps = videoFps, + }; + } + } + public class Config { public string SignalingUrl = ""; @@ -100,15 +138,9 @@ public class Config public SpotlightFocusRidType? SpotlightUnfocusRid; public bool? Simulcast; public SimulcastRidType? SimulcastRid = null; - public CapturerType CapturerType = Sora.CapturerType.DeviceCamera; - public UnityEngine.Camera UnityCamera = null; - public int UnityCameraRenderTargetDepthBuffer = 16; + public CameraConfig CameraConfig = new CameraConfig(); public bool Video = true; public bool Audio = true; - public string VideoCapturerDevice = ""; - public int VideoWidth = 640; - public int VideoHeight = 480; - public int VideoFps = 30; public VideoCodecType VideoCodecType = VideoCodecType.VP9; public string VideoVp9Params = ""; public string VideoAv1Params = ""; @@ -237,10 +269,10 @@ public Sora() public void Connect(Config config) { IntPtr unityCameraTexture = IntPtr.Zero; - if (config.CapturerType == CapturerType.UnityCamera) + if (config.CameraConfig.CapturerType == CapturerType.UnityCamera) { - unityCamera = config.UnityCamera; - var texture = new UnityEngine.RenderTexture(config.VideoWidth, config.VideoHeight, config.UnityCameraRenderTargetDepthBuffer, UnityEngine.RenderTextureFormat.BGRA32); + unityCamera = config.CameraConfig.UnityCamera; + var texture = new UnityEngine.RenderTexture(config.CameraConfig.VideoWidth, config.CameraConfig.VideoHeight, config.CameraConfig.UnityCameraRenderTargetDepthBuffer, UnityEngine.RenderTextureFormat.BGRA32); unityCamera.targetTexture = texture; unityCamera.enabled = true; unityCameraTexture = texture.GetNativeTexturePtr(); @@ -280,14 +312,14 @@ public void Connect(Config config) cc.simulcast = config.Simulcast == null ? false : config.Simulcast.Value; cc.simulcast_rid = config.SimulcastRid == null ? "" : config.SimulcastRid.Value.ToString().ToLower(); cc.insecure = config.Insecure; - cc.capturer_type = (int)config.CapturerType; - cc.unity_camera_texture = unityCameraTexture.ToInt64(); cc.video = config.Video; cc.audio = config.Audio; - cc.video_capturer_device = config.VideoCapturerDevice; - cc.video_width = config.VideoWidth; - cc.video_height = config.VideoHeight; - cc.video_fps = config.VideoFps; + cc.camera_config.capturer_type = (int)config.CameraConfig.CapturerType; + cc.camera_config.unity_camera_texture = unityCameraTexture.ToInt64(); + cc.camera_config.video_capturer_device = config.CameraConfig.VideoCapturerDevice; + cc.camera_config.video_width = config.CameraConfig.VideoWidth; + cc.camera_config.video_height = config.CameraConfig.VideoHeight; + cc.camera_config.video_fps = config.CameraConfig.VideoFps; cc.video_codec_type = config.VideoCodecType.ToString(); cc.video_vp9_params = config.VideoVp9Params; cc.video_av1_params = config.VideoAv1Params; @@ -380,6 +412,34 @@ public void Disconnect() sora_disconnect(p); } + public void SwitchCamera(CameraConfig config) + { + if (unityCamera != null) + { + unityCamera.enabled = false; + unityCamera.targetTexture = null; + unityCamera = null; + } + + IntPtr unityCameraTexture = IntPtr.Zero; + if (config.CapturerType == CapturerType.UnityCamera) + { + unityCamera = config.UnityCamera; + var texture = new UnityEngine.RenderTexture(config.VideoWidth, config.VideoHeight, config.UnityCameraRenderTargetDepthBuffer, UnityEngine.RenderTextureFormat.BGRA32); + unityCamera.targetTexture = texture; + unityCamera.enabled = true; + unityCameraTexture = texture.GetNativeTexturePtr(); + } + var cc = new SoraConf.Internal.CameraConfig(); + cc.capturer_type = (int)config.CapturerType; + cc.unity_camera_texture = unityCameraTexture.ToInt64(); + cc.video_capturer_device = config.VideoCapturerDevice; + cc.video_width = config.VideoWidth; + cc.video_height = config.VideoHeight; + cc.video_fps = config.VideoFps; + sora_switch_camera(p, Jsonif.Json.ToJson(cc)); + } + // Unity 側でレンダリングが完了した時（yield return new WaitForEndOfFrame() の後）に呼ぶイベント // 指定した Unity カメラの映像を Sora 側のテクスチャにレンダリングしたりする public void OnRender() @@ -791,6 +851,8 @@ public bool VideoEnabled [DllImport(DllName)] private static extern void sora_disconnect(IntPtr p); [DllImport(DllName)] + private static extern void sora_switch_camera(IntPtr p, string config); + [DllImport(DllName)] private static extern IntPtr sora_get_texture_update_callback(); [DllImport(DllName)] private static extern void sora_destroy(IntPtr p); diff --git a/proto/sora_conf_internal.proto b/proto/sora_conf_internal.proto index ed9c23b..83f86c7 100644 --- a/proto/sora_conf_internal.proto +++ b/proto/sora_conf_internal.proto @@ -37,6 +37,15 @@ message ForwardingFilter { repeated Rules rules = 2; } +message CameraConfig { + int32 capturer_type = 17; + int64 unity_camera_texture = 18; + string video_capturer_device = 19; + int32 video_width = 22; + int32 video_height = 23; + int32 video_fps = 24; +} + message ConnectConfig { string unity_version = 1; repeated string signaling_url = 2; @@ -54,14 +63,9 @@ message ConnectConfig { bool enable_simulcast = 14; bool simulcast = 15; string simulcast_rid = 16; - int32 capturer_type = 17; - int64 unity_camera_texture = 18; - string video_capturer_device = 19; + CameraConfig camera_config = 17; bool video = 20; bool audio = 21; - int32 video_width = 22; - int32 video_height = 23; - int32 video_fps = 24; string video_codec_type = 25; string video_vp9_params = 250; string video_av1_params = 251; diff --git a/src/sora.cpp b/src/sora.cpp index b6615b5..d6051c0 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -66,6 +66,7 @@ Sora::~Sora() { capturer_ = nullptr; unity_adm_ = nullptr; + video_sender_ = nullptr; audio_track_ = nullptr; video_track_ = nullptr; connection_context_ = nullptr; @@ -147,6 +148,40 @@ void Sora::DispatchEvents() { } } +static sora_conf::VideoFrame VideoFrameToConfig( + const webrtc::VideoFrame& frame) { + sora_conf::VideoFrame f; + f.baseptr = reinterpret_cast(&frame); + f.id = frame.id(); + f.timestamp_us = frame.timestamp_us(); + f.timestamp = frame.timestamp(); + f.ntp_time_ms = frame.ntp_time_ms(); + f.rotation = (int)frame.rotation(); + auto& v = f.video_frame_buffer; + auto vfb = frame.video_frame_buffer(); + v.baseptr = reinterpret_cast(vfb.get()); + v.type = (sora_conf::VideoFrameBuffer::Type)vfb->type(); + v.width = vfb->width(); + v.height = vfb->height(); + if (vfb->type() == webrtc::VideoFrameBuffer::Type::kI420) { + auto p = vfb->GetI420(); + v.i420_stride_y = p->StrideY(); + v.i420_stride_u = p->StrideU(); + v.i420_stride_v = p->StrideV(); + v.i420_data_y = reinterpret_cast(p->DataY()); + v.i420_data_u = reinterpret_cast(p->DataU()); + v.i420_data_v = reinterpret_cast(p->DataV()); + } + if (vfb->type() == webrtc::VideoFrameBuffer::Type::kNV12) { + auto p = vfb->GetNV12(); + v.nv12_stride_y = p->StrideY(); + v.nv12_stride_uv = p->StrideUV(); + v.nv12_data_y = reinterpret_cast(p->DataY()); + v.nv12_data_uv = reinterpret_cast(p->DataUV()); + } + return f; +} + void Sora::Connect(const sora_conf::internal::ConnectConfig& cc) { auto on_disconnect = [this](int error_code, std::string reason) { PushEvent([this, error_code, reason = std::move(reason)]() { @@ -302,43 +337,17 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, if (on_capturer_frame_) { on_frame = [on_frame = on_capturer_frame_](const webrtc::VideoFrame& frame) { - sora_conf::VideoFrame f; - f.baseptr = reinterpret_cast(&frame); - f.id = frame.id(); - f.timestamp_us = frame.timestamp_us(); - f.timestamp = frame.timestamp(); - f.ntp_time_ms = frame.ntp_time_ms(); - f.rotation = (int)frame.rotation(); - auto& v = f.video_frame_buffer; - auto vfb = frame.video_frame_buffer(); - v.baseptr = reinterpret_cast(vfb.get()); - v.type = (sora_conf::VideoFrameBuffer::Type)vfb->type(); - v.width = vfb->width(); - v.height = vfb->height(); - if (vfb->type() == webrtc::VideoFrameBuffer::Type::kI420) { - auto p = vfb->GetI420(); - v.i420_stride_y = p->StrideY(); - v.i420_stride_u = p->StrideU(); - v.i420_stride_v = p->StrideV(); - v.i420_data_y = reinterpret_cast(p->DataY()); - v.i420_data_u = reinterpret_cast(p->DataU()); - v.i420_data_v = reinterpret_cast(p->DataV()); - } - if (vfb->type() == webrtc::VideoFrameBuffer::Type::kNV12) { - auto p = vfb->GetNV12(); - v.nv12_stride_y = p->StrideY(); - v.nv12_stride_uv = p->StrideUV(); - v.nv12_data_y = reinterpret_cast(p->DataY()); - v.nv12_data_uv = reinterpret_cast(p->DataUV()); - } + sora_conf::VideoFrame f = VideoFrameToConfig(frame); on_frame(jsonif::to_json(f)); }; } auto capturer = CreateVideoCapturer( - cc.capturer_type, (void*)cc.unity_camera_texture, - cc.video_capturer_device, cc.video_width, cc.video_height, cc.video_fps, - on_frame, signaling_thread_.get(), env, android_context); + cc.camera_config.capturer_type, + (void*)cc.camera_config.unity_camera_texture, + cc.camera_config.video_capturer_device, cc.camera_config.video_width, + cc.camera_config.video_height, cc.camera_config.video_fps, on_frame, + signaling_thread_.get(), env, android_context); if (!capturer) { on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, "Capturer Init Failed"); @@ -346,7 +355,7 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, } capturer_ = capturer; - capturer_type_ = cc.capturer_type; + capturer_type_ = cc.camera_config.capturer_type; std::string audio_track_id = rtc::CreateRandomString(16); audio_track_ = factory_->CreateAudioTrack( @@ -530,6 +539,45 @@ void Sora::Disconnect() { signaling_->Disconnect(); } +void Sora::SwitchCamera(const sora_conf::internal::CameraConfig& cc) { + RTC_LOG(LS_INFO) << "SwitchCamera: " << jsonif::to_json(cc); + boost::asio::post(*ioc_, [self = shared_from_this(), cc = cc]() { + self->DoSwitchCamera(cc); + }); +} +void Sora::DoSwitchCamera(const sora_conf::internal::CameraConfig& cc) { + std::function on_frame; + if (on_capturer_frame_) { + on_frame = [on_frame = + on_capturer_frame_](const webrtc::VideoFrame& frame) { + sora_conf::VideoFrame f = VideoFrameToConfig(frame); + on_frame(jsonif::to_json(f)); + }; + } + + void* env = sora::GetJNIEnv(); + void* android_context = GetAndroidApplicationContext(env); + + capturer_ = nullptr; + auto capturer = CreateVideoCapturer( + cc.capturer_type, (void*)cc.unity_camera_texture, + cc.video_capturer_device, cc.video_width, cc.video_height, cc.video_fps, + on_frame, signaling_thread_.get(), env, android_context); + if (!capturer) { + RTC_LOG(LS_ERROR) << "Failed to CreateVideoCapturer"; + return; + } + + capturer_ = capturer; + capturer_type_ = cc.capturer_type; + + std::string video_track_id = rtc::CreateRandomString(16); + auto video_track = factory_->CreateVideoTrack(video_track_id, capturer.get()); + renderer_->ReplaceTrack(video_track_.get(), video_track.get()); + video_sender_->SetTrack(video_track.get()); + video_track_ = video_track; +} + void Sora::RenderCallbackStatic(int event_id) { auto sora = (Sora*)IdPointer::Instance().Lookup(event_id); if (sora == nullptr) { @@ -791,6 +839,7 @@ void Sora::OnSetOffer(std::string offer) { webrtc::RTCErrorOr> video_result = signaling_->GetPeerConnection()->AddTrack(video_track_, {stream_id}); + video_sender_ = video_result.value(); } if (video_track_ != nullptr) { diff --git a/src/sora.h b/src/sora.h index 84c2ae8..3b6d485 100644 --- a/src/sora.h +++ b/src/sora.h @@ -53,6 +53,7 @@ class Sora : public std::enable_shared_from_this, void Connect(const sora_conf::internal::ConnectConfig& cc); void Disconnect(); + void SwitchCamera(const sora_conf::internal::CameraConfig& cc); static void UNITY_INTERFACE_API RenderCallbackStatic(int event_id); int GetRenderCallbackEventID() const; @@ -91,6 +92,7 @@ class Sora : public std::enable_shared_from_this, private: void DoConnect(const sora_conf::internal::ConnectConfig& config, std::function on_disconnect); + void DoSwitchCamera(const sora_conf::internal::CameraConfig& cc); static rtc::scoped_refptr CreateADM( webrtc::TaskQueueFactory* task_queue_factory, @@ -140,6 +142,7 @@ class Sora : public std::enable_shared_from_this, std::unique_ptr renderer_; rtc::scoped_refptr audio_track_; rtc::scoped_refptr video_track_; + rtc::scoped_refptr video_sender_; std::function on_add_track_; std::function on_remove_track_; std::function on_set_offer_; diff --git a/src/unity.cpp b/src/unity.cpp index 3e6b72c..d4d01c0 100644 --- a/src/unity.cpp +++ b/src/unity.cpp @@ -142,6 +142,13 @@ void sora_disconnect(void* p) { wsora->sora->Disconnect(); } +void sora_switch_camera(void* p, const char* config_json) { + auto wsora = (SoraWrapper*)p; + auto config = + jsonif::from_json(config_json); + wsora->sora->SwitchCamera(config); +} + void* sora_get_texture_update_callback() { return (void*)&sora_unity_sdk::UnityRenderer::Sink::TextureUpdateCallback; } diff --git a/src/unity.h b/src/unity.h index 3007cf2..07fef2b 100644 --- a/src/unity.h +++ b/src/unity.h @@ -60,6 +60,7 @@ UNITY_INTERFACE_EXPORT void sora_set_on_capturer_frame(void* p, UNITY_INTERFACE_EXPORT void sora_dispatch_events(void* p); UNITY_INTERFACE_EXPORT void sora_connect(void* p, const char* config); UNITY_INTERFACE_EXPORT void sora_disconnect(void* p); +UNITY_INTERFACE_EXPORT void sora_switch_camera(void* p, const char* config); UNITY_INTERFACE_EXPORT void* sora_get_texture_update_callback(); UNITY_INTERFACE_EXPORT void sora_destroy(void* sora); diff --git a/src/unity_renderer.cpp b/src/unity_renderer.cpp index a3fddf7..c0c95fc 100644 --- a/src/unity_renderer.cpp +++ b/src/unity_renderer.cpp @@ -19,6 +19,11 @@ UnityRenderer::Sink::~Sink() { ptrid_t UnityRenderer::Sink::GetSinkID() const { return ptrid_; } +void UnityRenderer::Sink::SetTrack(webrtc::VideoTrackInterface* track) { + track_->RemoveSink(this); + track->AddOrUpdateSink(this, rtc::VideoSinkWants()); + track_ = track; +} rtc::scoped_refptr UnityRenderer::Sink::GetFrameBuffer() { @@ -107,4 +112,18 @@ ptrid_t UnityRenderer::RemoveTrack(webrtc::VideoTrackInterface* track) { return sink_id; } +void UnityRenderer::ReplaceTrack(webrtc::VideoTrackInterface* oldTrack, + webrtc::VideoTrackInterface* newTrack) { + RTC_LOG(LS_INFO) << "UnityRenderer::ReplaceTrack"; + auto it = std::find_if(sinks_.begin(), sinks_.end(), + [oldTrack](const VideoSinkVector::value_type& sink) { + return sink.first == oldTrack; + }); + if (it == sinks_.end()) { + return; + } + it->first = newTrack; + it->second->SetTrack(newTrack); +} + } // namespace sora_unity_sdk diff --git a/src/unity_renderer.h b/src/unity_renderer.h index 0349e43..44fe751 100644 --- a/src/unity_renderer.h +++ b/src/unity_renderer.h @@ -27,6 +27,7 @@ class UnityRenderer { Sink(webrtc::VideoTrackInterface* track); ~Sink(); ptrid_t GetSinkID() const; + void SetTrack(webrtc::VideoTrackInterface* track); private: rtc::scoped_refptr GetFrameBuffer(); @@ -47,6 +48,8 @@ class UnityRenderer { public: ptrid_t AddTrack(webrtc::VideoTrackInterface* track); ptrid_t RemoveTrack(webrtc::VideoTrackInterface* track); + void ReplaceTrack(webrtc::VideoTrackInterface* oldTrack, + webrtc::VideoTrackInterface* newTrack); }; } // namespace sora_unity_sdk From 3cf949d668f8cf7002702539942f7342d1636d41 Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 1 Aug 2023 03:58:34 +0900 Subject: [PATCH 02/42] =?UTF-8?q?=E3=82=AD=E3=83=A3=E3=83=97=E3=83=81?= =?UTF-8?q?=E3=83=A3=E3=83=A9=E3=81=AE=E7=B5=82=E4=BA=86=E5=87=A6=E7=90=86?= =?UTF-8?q?=E3=82=92=E5=85=A5=E3=82=8C=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sora.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/sora.cpp b/src/sora.cpp index d6051c0..d14ca65 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -558,7 +558,16 @@ void Sora::DoSwitchCamera(const sora_conf::internal::CameraConfig& cc) { void* env = sora::GetJNIEnv(); void* android_context = GetAndroidApplicationContext(env); +#if defined(SORA_UNITY_SDK_ANDROID) + if (capturer_ != nullptr && capturer_type_ == 0) { + static_cast(capturer_.get())->Stop(); + } +#endif + if (capturer_ != nullptr && capturer_type_ != 0) { + static_cast(capturer_.get())->Stop(); + } capturer_ = nullptr; + auto capturer = CreateVideoCapturer( cc.capturer_type, (void*)cc.unity_camera_texture, cc.video_capturer_device, cc.video_width, cc.video_height, cc.video_fps, From 00057a5495a517e1b0d8a01a961d07e54e836287 Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 1 Aug 2023 04:21:54 +0900 Subject: [PATCH 03/42] =?UTF-8?q?CHANGES=20=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 1cad635..a61f7d5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,12 @@ ## develop +- [CHANGE] `Sora.Config` 中にあるキャプチャラに関するフィールドを `Sora.CameraConfig` に移動する + - 修正方法は （TODO: @miosakuma がドキュメントへのリンクに差し替える） を参照して下さい + - @melpon +- [ADD] 接続中にキャプチャラを切り替える機能を実装 + - @melpon + ## 2023.2.0 (2023-07-19) - [UPDATE] libwebrtc を `m114.5735.2.0` に上げる From 3b21654a22cfad5b8c3979ca0012964cbc1aa8bf Mon Sep 17 00:00:00 2001 From: miosakuma Date: Tue, 8 Aug 2023 17:51:12 +0900 Subject: [PATCH 04/42] =?UTF-8?q?=E5=A4=89=E6=9B=B4=E5=B1=A5=E6=AD=B4?= =?UTF-8?q?=E3=82=92=E6=9B=B4=E6=96=B0=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 10dbf01..0042142 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,8 @@ - [UPDATE] Sora C++ SDK を `2023.9.0` に上げる - @torikizi +- [UPDATE] libwebrtc を `m115.5790.7.0` に上げる + - @torikizi ## 2023.2.0 (2023-07-19) From 0cd7fe103c990be5424ca572e3286fa71135e698 Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 9 Aug 2023 12:26:04 +0900 Subject: [PATCH 05/42] =?UTF-8?q?sora=5Fconf.proto=20=E3=81=AE=20enum=20Ty?= =?UTF-8?q?pe=20=E3=81=A8=20VideoFrameBuffer::Type=20=E3=81=AE=E5=80=A4?= =?UTF-8?q?=E3=81=8C=E4=B8=80=E8=87=B4=E3=81=97=E3=81=A6=E3=81=AA=E3=81=8B?= =?UTF-8?q?=E3=81=A3=E3=81=9F=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proto/sora_conf.proto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proto/sora_conf.proto b/proto/sora_conf.proto index 993b398..edc5e5f 100644 --- a/proto/sora_conf.proto +++ b/proto/sora_conf.proto @@ -33,7 +33,8 @@ message VideoFrameBuffer { kI444 = 4; kI010 = 5; kI210 = 6; - kNV12 = 7; + kI410 = 7; + kNV12 = 8; } int64 baseptr = 1; Type type = 2; From 05cf91c217335814eba9733dcc8fad38e0a2e064 Mon Sep 17 00:00:00 2001 From: voluntas Date: Fri, 25 Aug 2023 12:10:01 +0900 Subject: [PATCH 06/42] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index be9bf37..44df99c 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,7 @@ Please read https://github.com/shiguredo/oss/blob/master/README.en.md before use ### 機能 +- デバイス切り替え機能 - デバイスをつかまないようにする機能 - 音声のみ送受信機能 - サイマルキャスト rid 指定対応 From 090c913f6140983cfaef5bace06a9676ebc2a2b6 Mon Sep 17 00:00:00 2001 From: voluntas Date: Fri, 25 Aug 2023 16:41:23 +0900 Subject: [PATCH 07/42] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 44df99c..5b82c7b 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ Please read https://github.com/shiguredo/oss/blob/master/README.en.md before use ### 機能 -- デバイス切り替え機能 +- 音声出力先変更機能 - デバイスをつかまないようにする機能 - 音声のみ送受信機能 - サイマルキャスト rid 指定対応 From 78a0be35555689e211f2121756cb86601440a7da Mon Sep 17 00:00:00 2001 From: voluntas Date: Sat, 26 Aug 2023 21:13:52 +0900 Subject: [PATCH 08/42] =?UTF-8?q?C++=20SDK=20=E3=82=92=202023.10.0=20?= =?UTF-8?q?=E3=81=AB=E4=B8=8A=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 4 ++-- VERSION | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0042142..f95afa9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,8 +13,8 @@ ## 2023.3.0 (2023-08-08) -- [UPDATE] Sora C++ SDK を `2023.9.0` に上げる - - @torikizi +- [UPDATE] Sora C++ SDK を `2023.10.0` に上げる + - @torikizi @voluntas - [UPDATE] libwebrtc を `m115.5790.7.0` に上げる - @torikizi diff --git a/VERSION b/VERSION index 2cfe0d5..6c5721c 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ SORA_UNITY_SDK_VERSION=2023.3.0 -SORA_CPP_SDK_VERSION=2023.9.0 +SORA_CPP_SDK_VERSION=2023.10.0 WEBRTC_BUILD_VERSION=m115.5790.7.0 BOOST_VERSION=1.82.0 LYRA_VERSION=1.3.0 From 729cd9c467ded192b184a8d7d861666708e91193 Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 30 Aug 2023 23:21:57 +0900 Subject: [PATCH 09/42] =?UTF-8?q?SoraClientContext=20=E3=82=92=E5=88=A9?= =?UTF-8?q?=E7=94=A8=E3=81=97=E3=81=A6=E3=82=B3=E3=83=BC=E3=83=89=E3=82=92?= =?UTF-8?q?=E7=9F=AD=E3=81=8F=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 3 + src/sora.cpp | 158 ++++++++++++++++++--------------------------------- src/sora.h | 12 ++-- 3 files changed, 62 insertions(+), 111 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f95afa9..010ca61 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,9 @@ ## develop +- [UPDATE] SoraClientContext を利用してコードを短くする + - @melpon + ## 2023.3.0 (2023-08-08) - [UPDATE] Sora C++ SDK を `2023.10.0` に上げる diff --git a/src/sora.cpp b/src/sora.cpp index b6615b5..64aa201 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -39,7 +39,7 @@ namespace sora_unity_sdk { -Sora::Sora(UnityContext* context) : context_(context) { +Sora::Sora(UnityContext* context) : unity_context_(context) { ptrid_ = IdPointer::Instance().Register(this); #if defined(SORA_UNITY_SDK_ANDROID) auto env = sora::GetJNIEnv(); @@ -68,8 +68,6 @@ Sora::~Sora() { audio_track_ = nullptr; video_track_ = nullptr; - connection_context_ = nullptr; - factory_ = nullptr; if (ioc_ != nullptr) { ioc_->stop(); @@ -80,18 +78,7 @@ Sora::~Sora() { io_thread_->Stop(); io_thread_.reset(); } - if (network_thread_) { - network_thread_->Stop(); - network_thread_.reset(); - } - if (worker_thread_) { - worker_thread_->Stop(); - worker_thread_.reset(); - } - if (signaling_thread_) { - signaling_thread_->Stop(); - signaling_thread_.reset(); - } + sora_context_ = nullptr; RTC_LOG(LS_INFO) << "Sora object destroy finished"; } @@ -196,98 +183,53 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, return; } - rtc::InitializeSSL(); - - network_thread_ = rtc::Thread::CreateWithSocketServer(); - network_thread_->Start(); - worker_thread_ = rtc::Thread::Create(); - worker_thread_->Start(); - signaling_thread_ = rtc::Thread::Create(); - signaling_thread_->Start(); - - webrtc::PeerConnectionFactoryDependencies dependencies; - dependencies.network_thread = network_thread_.get(); - dependencies.worker_thread = worker_thread_.get(); - dependencies.signaling_thread = signaling_thread_.get(); - dependencies.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(); - dependencies.call_factory = webrtc::CreateCallFactory(); - dependencies.event_log_factory = - absl::make_unique( - dependencies.task_queue_factory.get()); - - // worker 上の env - auto worker_env = - worker_thread_->BlockingCall([] { return sora::GetJNIEnv(); }); - // worker 上の context + // このスレッド上の env と context + void* env = sora::GetJNIEnv(); + void* android_context = GetAndroidApplicationContext(env); + + sora::SoraClientContextConfig client_config; + client_config.use_audio_device = false; + client_config.configure_media_dependencies = + [&, this](const webrtc::PeerConnectionFactoryDependencies& dependencies, + cricket::MediaEngineDependencies& media_dependencies) { + // worker 上の env + auto worker_env = dependencies.worker_thread->BlockingCall( + [] { return sora::GetJNIEnv(); }); + // worker 上の context #if defined(SORA_UNITY_SDK_ANDROID) - void* worker_context = worker_thread_->BlockingCall([worker_env] { - return ::sora_unity_sdk::GetAndroidApplicationContext((JNIEnv*)worker_env) - .Release(); - }); + void* worker_context = + dependencies.worker_thread->BlockingCall([worker_env] { + return ::sora_unity_sdk::GetAndroidApplicationContext( + (JNIEnv*)worker_env) + .Release(); + }); #else - void* worker_context = nullptr; + void* worker_context = nullptr; #endif - // media_dependencies - cricket::MediaEngineDependencies media_dependencies; - media_dependencies.task_queue_factory = dependencies.task_queue_factory.get(); - unity_adm_ = - CreateADM(media_dependencies.task_queue_factory, false, - cc.unity_audio_input, cc.unity_audio_output, on_handle_audio_, - cc.audio_recording_device, cc.audio_playout_device, - worker_thread_.get(), worker_env, worker_context); - media_dependencies.adm = unity_adm_; + unity_adm_ = CreateADM( + media_dependencies.task_queue_factory, false, cc.unity_audio_input, + cc.unity_audio_output, on_handle_audio_, cc.audio_recording_device, + cc.audio_playout_device, dependencies.worker_thread, worker_env, + worker_context); + media_dependencies.adm = unity_adm_; #if defined(SORA_UNITY_SDK_ANDROID) - worker_thread_->BlockingCall([worker_env, worker_context] { - ((JNIEnv*)worker_env)->DeleteLocalRef((jobject)worker_context); - }); + dependencies.worker_thread->BlockingCall([worker_env, worker_context] { + ((JNIEnv*)worker_env)->DeleteLocalRef((jobject)worker_context); + }); #endif + }; - media_dependencies.audio_encoder_factory = - sora::CreateBuiltinAudioEncoderFactory(); - media_dependencies.audio_decoder_factory = - sora::CreateBuiltinAudioDecoderFactory(); - - void* env = sora::GetJNIEnv(); - void* android_context = GetAndroidApplicationContext(env); - - auto cuda_context = sora::CudaContext::Create(); - { - auto config = sora::GetDefaultVideoEncoderFactoryConfig(cuda_context, env); - config.use_simulcast_adapter = true; - media_dependencies.video_encoder_factory = - absl::make_unique(std::move(config)); - } - { - auto config = sora::GetDefaultVideoDecoderFactoryConfig(cuda_context, env); - media_dependencies.video_decoder_factory = - absl::make_unique(std::move(config)); - } - - media_dependencies.audio_mixer = nullptr; - media_dependencies.audio_processing = - webrtc::AudioProcessingBuilder().Create(); - - dependencies.media_engine = - cricket::CreateMediaEngine(std::move(media_dependencies)); - - factory_ = sora::CreateModularPeerConnectionFactoryWithContext( - std::move(dependencies), connection_context_); + sora_context_ = sora::SoraClientContext::Create(client_config); - if (factory_ == nullptr) { + if (sora_context_ == nullptr) { RTC_LOG(LS_ERROR) << "Failed to create PeerConnectionFactory"; on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, "Failed to create PeerConnectionFactory"); return; } - webrtc::PeerConnectionFactoryInterface::Options factory_options; - factory_options.disable_encryption = false; - factory_options.ssl_max_version = rtc::SSL_PROTOCOL_DTLS_12; - factory_options.crypto_options.srtp.enable_gcm_crypto_suites = true; - factory_->SetOptions(factory_options); - if (!InitADM(unity_adm_, cc.audio_recording_device, cc.audio_playout_device)) { on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, @@ -338,7 +280,8 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, auto capturer = CreateVideoCapturer( cc.capturer_type, (void*)cc.unity_camera_texture, cc.video_capturer_device, cc.video_width, cc.video_height, cc.video_fps, - on_frame, signaling_thread_.get(), env, android_context); + on_frame, sora_context_->signaling_thread(), env, android_context, + unity_context_); if (!capturer) { on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, "Capturer Init Failed"); @@ -349,11 +292,13 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, capturer_type_ = cc.capturer_type; std::string audio_track_id = rtc::CreateRandomString(16); - audio_track_ = factory_->CreateAudioTrack( - audio_track_id, - factory_->CreateAudioSource(cricket::AudioOptions()).get()); + audio_track_ = sora_context_->peer_connection_factory()->CreateAudioTrack( + audio_track_id, sora_context_->peer_connection_factory() + ->CreateAudioSource(cricket::AudioOptions()) + .get()); std::string video_track_id = rtc::CreateRandomString(16); - video_track_ = factory_->CreateVideoTrack(video_track_id, capturer.get()); + video_track_ = sora_context_->peer_connection_factory()->CreateVideoTrack( + video_track_id, capturer.get()); } { @@ -361,7 +306,7 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, ioc_.reset(new boost::asio::io_context(1)); sora::SoraSignalingConfig config; config.observer = shared_from_this(); - config.pc_factory = factory_; + config.pc_factory = sora_context_->peer_connection_factory(); config.io_context = ioc_.get(); config.role = cc.role; config.sora_client = SORA_CLIENT; @@ -493,10 +438,14 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, } config.forwarding_filter = ff; } - config.network_manager = signaling_thread_->BlockingCall( - [this]() { return connection_context_->default_network_manager(); }); - config.socket_factory = signaling_thread_->BlockingCall( - [this]() { return connection_context_->default_socket_factory(); }); + config.network_manager = + sora_context_->signaling_thread()->BlockingCall([this]() { + return sora_context_->connection_context()->default_network_manager(); + }); + config.socket_factory = + sora_context_->signaling_thread()->BlockingCall([this]() { + return sora_context_->connection_context()->default_socket_factory(); + }); signaling_ = sora::SoraSignaling::Create(std::move(config)); signaling_->Connect(); @@ -684,7 +633,8 @@ rtc::scoped_refptr Sora::CreateVideoCapturer( std::function on_frame, rtc::Thread* signaling_thread, void* jni_env, - void* android_context) { + void* android_context, + UnityContext* unity_context) { if (capturer_type == 0) { // 実カメラ（デバイス）を使う sora::CameraDeviceCapturerConfig config; @@ -701,7 +651,7 @@ rtc::scoped_refptr Sora::CreateVideoCapturer( } else { // Unity のカメラからの映像を使う UnityCameraCapturerConfig config; - config.context = &UnityContext::Instance(); + config.context = unity_context; config.unity_camera_texture = unity_camera_texture; config.width = video_width; config.height = video_height; diff --git a/src/sora.h b/src/sora.h index 84c2ae8..8935e3b 100644 --- a/src/sora.h +++ b/src/sora.h @@ -6,6 +6,7 @@ #include // Sora +#include #include // Boost @@ -118,7 +119,8 @@ class Sora : public std::enable_shared_from_this, std::function on_frame, rtc::Thread* signaling_thread, void* jni_env, - void* android_context); + void* android_context, + UnityContext* unity_context); void PushEvent(std::function f); @@ -136,7 +138,7 @@ class Sora : public std::enable_shared_from_this, private: std::unique_ptr ioc_; std::shared_ptr signaling_; - UnityContext* context_; + UnityContext* unity_context_; std::unique_ptr renderer_; rtc::scoped_refptr audio_track_; rtc::scoped_refptr video_track_; @@ -151,12 +153,8 @@ class Sora : public std::enable_shared_from_this, std::function on_handle_audio_; std::function on_capturer_frame_; + std::shared_ptr sora_context_; std::unique_ptr io_thread_; - std::unique_ptr network_thread_; - std::unique_ptr worker_thread_; - std::unique_ptr signaling_thread_; - rtc::scoped_refptr factory_; - rtc::scoped_refptr connection_context_; std::mutex event_mutex_; std::deque> event_queue_; From 3637e4ed1114b55f7b9eb5a1b71a7474aa628491 Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 30 Aug 2023 23:38:32 +0900 Subject: [PATCH 10/42] =?UTF-8?q?=E3=83=8F=E3=83=BC=E3=83=89=E3=82=A6?= =?UTF-8?q?=E3=82=A7=E3=82=A2=E3=82=A8=E3=83=B3=E3=82=B3=E3=83=BC=E3=83=80?= =?UTF-8?q?=E3=82=92=E5=88=A9=E7=94=A8=E3=81=99=E3=82=8B=E3=81=8B=E3=81=A9?= =?UTF-8?q?=E3=81=86=E3=81=8B=E3=82=92=E8=A8=AD=E5=AE=9A=E3=81=99=E3=82=8B?= =?UTF-8?q?=20`UseHardwareEncoder`=20=E3=82=AA=E3=83=97=E3=82=B7=E3=83=A7?= =?UTF-8?q?=E3=83=B3=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 ++ Sora/Sora.cs | 12 ++++++++---- proto/sora_conf_internal.proto | 2 ++ src/sora.cpp | 1 + 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 010ca61..b7d04f2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,8 @@ ## develop +- [ADD] ハードウェアエンコーダを利用するかどうかを設定する `UseHardwareEncoder` オプションを追加 + - @melpon - [UPDATE] SoraClientContext を利用してコードを短くする - @melpon diff --git a/Sora/Sora.cs b/Sora/Sora.cs index c48b5c8..e90ab1f 100644 --- a/Sora/Sora.cs +++ b/Sora/Sora.cs @@ -153,6 +153,8 @@ public class Config public string ProxyAgent = ""; public ForwardingFilter ForwardingFilter; + + public bool? UseHardwareEncoder; } IntPtr p; @@ -270,14 +272,14 @@ public void Connect(Config config) cc.signaling_notify_metadata = config.SignalingNotifyMetadata; cc.role = role; cc.enable_multistream = config.Multistream != null; - cc.multistream = config.Multistream == null ? false : config.Multistream.Value; + cc.multistream = config.Multistream.GetValueOrDefault(); cc.enable_spotlight = config.Spotlight != null; - cc.spotlight = config.Spotlight == null ? false : config.Spotlight.Value; + cc.spotlight = config.Spotlight.GetValueOrDefault(); cc.spotlight_number = config.SpotlightNumber; cc.spotlight_focus_rid = config.SpotlightFocusRid == null ? "" : config.SpotlightFocusRid.Value.ToString().ToLower(); cc.spotlight_unfocus_rid = config.SpotlightUnfocusRid == null ? "" : config.SpotlightUnfocusRid.Value.ToString().ToLower(); cc.enable_simulcast = config.Simulcast != null; - cc.simulcast = config.Simulcast == null ? false : config.Simulcast.Value; + cc.simulcast = config.Simulcast.GetValueOrDefault(); cc.simulcast_rid = config.SimulcastRid == null ? "" : config.SimulcastRid.Value.ToString().ToLower(); cc.insecure = config.Insecure; cc.capturer_type = (int)config.CapturerType; @@ -300,7 +302,7 @@ public void Connect(Config config) cc.audio_codec_type = config.AudioCodecType.ToString(); cc.audio_codec_lyra_bitrate = config.AudioCodecLyraBitrate; cc.enable_audio_codec_lyra_usedtx = config.AudioCodecLyraUsedtx != null; - cc.audio_codec_lyra_usedtx = config.AudioCodecLyraUsedtx == null ? false : config.AudioCodecLyraUsedtx.Value; + cc.audio_codec_lyra_usedtx = config.AudioCodecLyraUsedtx.GetValueOrDefault(); cc.check_lyra_version = config.CheckLyraVersion; cc.audio_bit_rate = config.AudioBitRate; cc.audio_streaming_language_code = config.AudioStreamingLanguageCode; @@ -372,6 +374,8 @@ public void Connect(Config config) cc.forwarding_filter.rules.Add(ccrs); } } + cc.enable_use_hardware_encoder = config.UseHardwareEncoder != null; + cc.use_hardware_encoder = config.UseHardwareEncoder.GetValueOrDefault(); sora_connect(p, Jsonif.Json.ToJson(cc)); } diff --git a/proto/sora_conf_internal.proto b/proto/sora_conf_internal.proto index ed9c23b..59c69d4 100644 --- a/proto/sora_conf_internal.proto +++ b/proto/sora_conf_internal.proto @@ -94,4 +94,6 @@ message ConnectConfig { string signaling_notify_metadata = 49; bool enable_forwarding_filter = 50; ForwardingFilter forwarding_filter = 51; + bool enable_use_hardware_encoder = 52; + bool use_hardware_encoder = 53; } \ No newline at end of file diff --git a/src/sora.cpp b/src/sora.cpp index 64aa201..207f797 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -189,6 +189,7 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, sora::SoraClientContextConfig client_config; client_config.use_audio_device = false; + client_config.use_hardware_encoder = cc.use_hardware_encoder; client_config.configure_media_dependencies = [&, this](const webrtc::PeerConnectionFactoryDependencies& dependencies, cricket::MediaEngineDependencies& media_dependencies) { From 4ea9ecf9972fb332be1a6f89cc900f1ba5adc484 Mon Sep 17 00:00:00 2001 From: melpon Date: Thu, 31 Aug 2023 00:53:05 +0900 Subject: [PATCH 11/42] =?UTF-8?q?=E3=83=87=E3=83=90=E3=82=A4=E3=82=B9?= =?UTF-8?q?=E3=82=92=E6=8E=B4=E3=81=BE=E3=81=AA=E3=81=84=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E3=81=99=E3=82=8B=20`NoVideoDevice`,=20`NoAudioDevice?= =?UTF-8?q?`=20=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 4 +++- Sora/Sora.cs | 4 ++++ proto/sora_conf_internal.proto | 2 ++ src/sora.cpp | 26 +++++++++++++++++--------- src/sora.h | 1 + 5 files changed, 27 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b7d04f2..ed676de 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,7 +11,9 @@ ## develop -- [ADD] ハードウェアエンコーダを利用するかどうかを設定する `UseHardwareEncoder` オプションを追加 +- [ADD] デバイスを掴まないようにする `NoVideoDevice`, `NoAudioDevice` を追加 + - @melpon +- [ADD] ハードウェアエンコーダを利用するかどうかを設定する `UseHardwareEncoder` を追加 - @melpon - [UPDATE] SoraClientContext を利用してコードを短くする - @melpon diff --git a/Sora/Sora.cs b/Sora/Sora.cs index e90ab1f..669197e 100644 --- a/Sora/Sora.cs +++ b/Sora/Sora.cs @@ -100,6 +100,8 @@ public class Config public SpotlightFocusRidType? SpotlightUnfocusRid; public bool? Simulcast; public SimulcastRidType? SimulcastRid = null; + public bool NoVideoDevice = false; + public bool NoAudioDevice = false; public CapturerType CapturerType = Sora.CapturerType.DeviceCamera; public UnityEngine.Camera UnityCamera = null; public int UnityCameraRenderTargetDepthBuffer = 16; @@ -282,6 +284,8 @@ public void Connect(Config config) cc.simulcast = config.Simulcast.GetValueOrDefault(); cc.simulcast_rid = config.SimulcastRid == null ? "" : config.SimulcastRid.Value.ToString().ToLower(); cc.insecure = config.Insecure; + cc.no_video_device = config.NoVideoDevice; + cc.no_audio_device = config.NoAudioDevice; cc.capturer_type = (int)config.CapturerType; cc.unity_camera_texture = unityCameraTexture.ToInt64(); cc.video = config.Video; diff --git a/proto/sora_conf_internal.proto b/proto/sora_conf_internal.proto index 59c69d4..4cc1120 100644 --- a/proto/sora_conf_internal.proto +++ b/proto/sora_conf_internal.proto @@ -54,6 +54,8 @@ message ConnectConfig { bool enable_simulcast = 14; bool simulcast = 15; string simulcast_rid = 16; + bool no_video_device = 160; + bool no_audio_device = 161; int32 capturer_type = 17; int64 unity_camera_texture = 18; string video_capturer_device = 19; diff --git a/src/sora.cpp b/src/sora.cpp index 207f797..1621341 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -209,10 +209,10 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, #endif unity_adm_ = CreateADM( - media_dependencies.task_queue_factory, false, cc.unity_audio_input, - cc.unity_audio_output, on_handle_audio_, cc.audio_recording_device, - cc.audio_playout_device, dependencies.worker_thread, worker_env, - worker_context); + media_dependencies.task_queue_factory, cc.no_audio_device, + cc.unity_audio_input, cc.unity_audio_output, on_handle_audio_, + cc.audio_recording_device, cc.audio_playout_device, + dependencies.worker_thread, worker_env, worker_context); media_dependencies.adm = unity_adm_; #if defined(SORA_UNITY_SDK_ANDROID) @@ -279,11 +279,11 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, } auto capturer = CreateVideoCapturer( - cc.capturer_type, (void*)cc.unity_camera_texture, + cc.capturer_type, (void*)cc.unity_camera_texture, cc.no_video_device, cc.video_capturer_device, cc.video_width, cc.video_height, cc.video_fps, on_frame, sora_context_->signaling_thread(), env, android_context, unity_context_); - if (!capturer) { + if (!cc.no_video_device && !capturer) { on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, "Capturer Init Failed"); return; @@ -297,9 +297,12 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, audio_track_id, sora_context_->peer_connection_factory() ->CreateAudioSource(cricket::AudioOptions()) .get()); - std::string video_track_id = rtc::CreateRandomString(16); - video_track_ = sora_context_->peer_connection_factory()->CreateVideoTrack( - video_track_id, capturer.get()); + + if (capturer) { + std::string video_track_id = rtc::CreateRandomString(16); + video_track_ = sora_context_->peer_connection_factory()->CreateVideoTrack( + video_track_id, capturer.get()); + } } { @@ -627,6 +630,7 @@ bool Sora::InitADM(rtc::scoped_refptr adm, rtc::scoped_refptr Sora::CreateVideoCapturer( int capturer_type, void* unity_camera_texture, + bool no_video_device, std::string video_capturer_device, int video_width, int video_height, @@ -637,6 +641,10 @@ rtc::scoped_refptr Sora::CreateVideoCapturer( void* android_context, UnityContext* unity_context) { if (capturer_type == 0) { + if (no_video_device) { + return nullptr; + } + // 実カメラ（デバイス）を使う sora::CameraDeviceCapturerConfig config; config.width = video_width; diff --git a/src/sora.h b/src/sora.h index 8935e3b..c570ea6 100644 --- a/src/sora.h +++ b/src/sora.h @@ -112,6 +112,7 @@ class Sora : public std::enable_shared_from_this, CreateVideoCapturer( int capturer_type, void* unity_camera_texture, + bool no_video_device, std::string video_capturer_device, int video_width, int video_height, From a3d172d759a1dcae33ebf534ccaffeba397f30c9 Mon Sep 17 00:00:00 2001 From: melpon Date: Thu, 31 Aug 2023 23:53:50 +0900 Subject: [PATCH 12/42] =?UTF-8?q?=E5=88=9D=E5=9B=9E=E3=81=AE=E3=81=BF=20Io?= =?UTF-8?q?sAudioInit=20=E3=82=92=E5=91=BC=E3=81=B6=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sora.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/sora.cpp b/src/sora.cpp index 1621341..a880340 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -152,14 +152,18 @@ void Sora::Connect(const sora_conf::internal::ConnectConfig& cc) { return; } - IosAudioInit( - [this, on_disconnect = std::move(on_disconnect)](std::string error) { - if (!error.empty()) { - RTC_LOG(LS_ERROR) << "Failed to IosAudioInit: error=" << error; - on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, - "Failed to IosAudioInit: error=" + error); - } - }); + static bool ios_audio_init = false; + if (!ios_audio_init) { + IosAudioInit( + [this, on_disconnect = std::move(on_disconnect)](std::string error) { + if (!error.empty()) { + RTC_LOG(LS_ERROR) << "Failed to IosAudioInit: error=" << error; + on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, + "Failed to IosAudioInit: error=" + error); + } + ios_audio_init = true; + }); + } DoConnect(cc, std::move(on_disconnect)); #else DoConnect(cc, std::move(on_disconnect)); From 5a6634c5ff8dec3d6747e2b07b8e57a1dbd5fddc Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 6 Sep 2023 02:43:44 +0900 Subject: [PATCH 13/42] =?UTF-8?q?=E3=83=90=E3=83=BC=E3=82=B8=E3=83=A7?= =?UTF-8?q?=E3=83=B3=E3=81=AE=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 8 ++++++-- VERSION | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ed676de..4b26c49 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,12 +11,16 @@ ## develop +- [UPDATE] SoraClientContext を利用してコードを短くする + - @melpon +- [UPDATE] Sora C++ SDK を `2023.11.0` に上げる + - @melpon +- [UPDATE] libwebrtc を `m116.5845.6.1` に上げる + - @melpon - [ADD] デバイスを掴まないようにする `NoVideoDevice`, `NoAudioDevice` を追加 - @melpon - [ADD] ハードウェアエンコーダを利用するかどうかを設定する `UseHardwareEncoder` を追加 - @melpon -- [UPDATE] SoraClientContext を利用してコードを短くする - - @melpon ## 2023.3.0 (2023-08-08) diff --git a/VERSION b/VERSION index 6c5721c..9a61b40 100644 --- a/VERSION +++ b/VERSION @@ -1,6 +1,6 @@ SORA_UNITY_SDK_VERSION=2023.3.0 -SORA_CPP_SDK_VERSION=2023.10.0 -WEBRTC_BUILD_VERSION=m115.5790.7.0 +SORA_CPP_SDK_VERSION=2023.11.0 +WEBRTC_BUILD_VERSION=m116.5845.6.1 BOOST_VERSION=1.82.0 LYRA_VERSION=1.3.0 CMAKE_VERSION=3.26.4 From 4251a9118a26e79d554945f7f19e65c911fc4699 Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 6 Sep 2023 04:08:50 +0900 Subject: [PATCH 14/42] =?UTF-8?q?`SelectedSignalingURL`=20=E3=81=A8=20`Con?= =?UTF-8?q?nectedSignalingURL`=20=E3=83=97=E3=83=AD=E3=83=91=E3=83=86?= =?UTF-8?q?=E3=82=A3=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 ++ Sora/Sora.cs | 30 ++++++++++++++++++++++++++++++ src/sora.cpp | 14 ++++++++++++++ src/sora.h | 3 +++ src/unity.cpp | 21 +++++++++++++++++++++ src/unity.h | 9 +++++++++ 6 files changed, 79 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 4b26c49..68844f1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,8 @@ - @melpon - [ADD] ハードウェアエンコーダを利用するかどうかを設定する `UseHardwareEncoder` を追加 - @melpon +- [ADD] `SelectedSignalingURL` と `ConnectedSignalingURL` プロパティを追加 + - @melpon ## 2023.3.0 (2023-08-08) diff --git a/Sora/Sora.cs b/Sora/Sora.cs index 669197e..c5a68d0 100644 --- a/Sora/Sora.cs +++ b/Sora/Sora.cs @@ -768,6 +768,28 @@ public bool VideoEnabled set { sora_set_video_enabled(p, value ? 1 : 0); } } + public string SelectedSignalingURL + { + get + { + int size = sora_get_selected_signaling_url_size(p); + byte[] buf = new byte[size]; + sora_get_selected_signaling_url(p, buf, size); + return System.Text.Encoding.UTF8.GetString(buf); + } + } + + public string ConnectedSignalingURL + { + get + { + int size = sora_get_connected_signaling_url_size(p); + byte[] buf = new byte[size]; + sora_get_connected_signaling_url(p, buf, size); + return System.Text.Encoding.UTF8.GetString(buf); + } + } + #if UNITY_IOS && !UNITY_EDITOR private const string DllName = "__Internal"; #else @@ -834,4 +856,12 @@ public bool VideoEnabled private static extern int sora_get_video_enabled(IntPtr p); [DllImport(DllName)] private static extern void sora_set_video_enabled(IntPtr p, int enabled); + [DllImport(DllName)] + private static extern int sora_get_selected_signaling_url_size(IntPtr p); + [DllImport(DllName)] + private static extern int sora_get_connected_signaling_url_size(IntPtr p); + [DllImport(DllName)] + private static extern void sora_get_selected_signaling_url(IntPtr p, [Out] byte[] buf, int size); + [DllImport(DllName)] + private static extern void sora_get_connected_signaling_url(IntPtr p, [Out] byte[] buf, int size); } diff --git a/src/sora.cpp b/src/sora.cpp index a880340..c27352e 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -871,4 +871,18 @@ void Sora::SetVideoEnabled(bool enabled) { video_track_->set_enabled(enabled); } +std::string Sora::GetSelectedSignalingURL() const { + if (signaling_ == nullptr) { + return ""; + } + return signaling_->GetSelectedSignalingURL(); +} + +std::string Sora::GetConnectedSignalingURL() const { + if (signaling_ == nullptr) { + return ""; + } + return signaling_->GetConnectedSignalingURL(); +} + } // namespace sora_unity_sdk diff --git a/src/sora.h b/src/sora.h index c570ea6..d928ab6 100644 --- a/src/sora.h +++ b/src/sora.h @@ -72,6 +72,9 @@ class Sora : public std::enable_shared_from_this, bool GetVideoEnabled() const; void SetVideoEnabled(bool enabled); + std::string GetSelectedSignalingURL() const; + std::string GetConnectedSignalingURL() const; + private: void* GetAndroidApplicationContext(void* env); static sora_conf::ErrorCode ToErrorCode(sora::SoraSignalingErrorCode ec); diff --git a/src/unity.cpp b/src/unity.cpp index 3e6b72c..e332927 100644 --- a/src/unity.cpp +++ b/src/unity.cpp @@ -246,6 +246,27 @@ void sora_set_video_enabled(void* p, unity_bool_t enabled) { wsora->sora->SetVideoEnabled(enabled); } +// get_*_signaling_url_size() から get_*_signaling_url() までの間に値が変わった場合、 +// 落ちることは無いが、文字列が切り詰められる可能性があるので注意 +int sora_get_selected_signaling_url_size(void* p) { + auto wsora = (SoraWrapper*)p; + return wsora->sora->GetSelectedSignalingURL().size(); +} +int sora_get_connected_signaling_url_size(void* p) { + auto wsora = (SoraWrapper*)p; + return wsora->sora->GetConnectedSignalingURL().size(); +} +void sora_get_selected_signaling_url(void* p, void* buf, int size) { + auto wsora = (SoraWrapper*)p; + std::string str = wsora->sora->GetSelectedSignalingURL(); + std::memcpy(buf, str.c_str(), std::min(size, (int)str.size())); +} +void sora_get_connected_signaling_url(void* p, void* buf, int size) { + auto wsora = (SoraWrapper*)p; + std::string str = wsora->sora->GetConnectedSignalingURL(); + std::memcpy(buf, str.c_str(), std::min(size, (int)str.size())); +} + // iOS の場合は static link で名前が被る可能性があるので、別の名前にしておく void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API #if defined(SORA_UNITY_SDK_IOS) diff --git a/src/unity.h b/src/unity.h index 3007cf2..cc55467 100644 --- a/src/unity.h +++ b/src/unity.h @@ -106,6 +106,15 @@ UNITY_INTERFACE_EXPORT unity_bool_t sora_get_video_enabled(void* p); UNITY_INTERFACE_EXPORT void sora_set_video_enabled(void* p, unity_bool_t enabled); +UNITY_INTERFACE_EXPORT int sora_get_selected_signaling_url_size(void* p); +UNITY_INTERFACE_EXPORT int sora_get_connected_signaling_url_size(void* p); +UNITY_INTERFACE_EXPORT void sora_get_selected_signaling_url(void* p, + void* buf, + int size); +UNITY_INTERFACE_EXPORT void sora_get_connected_signaling_url(void* p, + void* buf, + int size); + #ifdef __cplusplus } #endif From 492ceb876f925eccb4346582e2ba8d6534d7e85b Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 6 Sep 2023 04:11:39 +0900 Subject: [PATCH 15/42] =?UTF-8?q?CHANGES=20=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 68844f1..c8fb531 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -23,6 +23,8 @@ - @melpon - [ADD] `SelectedSignalingURL` と `ConnectedSignalingURL` プロパティを追加 - @melpon +- [FIX] IosAudioInit を初回接続の場合のみ呼び出すようにすることで、iOS で連続して接続しようとすると落ちることがあったのを修正 + - @melpon ## 2023.3.0 (2023-08-08) From f5decce0f7cc6ce8ba21a3c1ed5a85640a40acfa Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 6 Sep 2023 04:47:03 +0900 Subject: [PATCH 16/42] =?UTF-8?q?=E3=82=B3=E3=83=B3=E3=83=91=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E3=82=A8=E3=83=A9=E3=83=BC=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sora.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/sora.cpp b/src/sora.cpp index 291de08..1572ed6 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -294,7 +294,8 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, (void*)cc.camera_config.unity_camera_texture, cc.no_video_device, cc.camera_config.video_capturer_device, cc.camera_config.video_width, cc.camera_config.video_height, cc.camera_config.video_fps, on_frame, - signaling_thread_.get(), env, android_context); + sora_context_->signaling_thread(), env, android_context, + unity_context_); if (!cc.no_video_device && !capturer) { on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, "Capturer Init Failed"); @@ -527,7 +528,8 @@ void Sora::DoSwitchCamera(const sora_conf::internal::CameraConfig& cc) { auto capturer = CreateVideoCapturer( cc.capturer_type, (void*)cc.unity_camera_texture, false, cc.video_capturer_device, cc.video_width, cc.video_height, cc.video_fps, - on_frame, signaling_thread_.get(), env, android_context); + on_frame, sora_context_->signaling_thread(), env, android_context, + unity_context_); if (!capturer) { RTC_LOG(LS_ERROR) << "Failed to CreateVideoCapturer"; return; @@ -537,7 +539,8 @@ void Sora::DoSwitchCamera(const sora_conf::internal::CameraConfig& cc) { capturer_type_ = cc.capturer_type; std::string video_track_id = rtc::CreateRandomString(16); - auto video_track = factory_->CreateVideoTrack(video_track_id, capturer.get()); + auto video_track = sora_context_->peer_connection_factory()->CreateVideoTrack( + video_track_id, capturer.get()); if (video_track_ == nullptr) { auto track_id = renderer_->AddTrack(video_track.get()); PushEvent([this, track_id]() { From f9e76924720bc2ed47576283298bd6ac6bf3a364 Mon Sep 17 00:00:00 2001 From: melpon Date: Thu, 7 Sep 2023 15:50:13 +0900 Subject: [PATCH 17/42] =?UTF-8?q?=E3=82=AD=E3=83=A3=E3=83=97=E3=83=81?= =?UTF-8?q?=E3=83=A3=E3=83=A9=E3=81=AE=E4=BD=9C=E6=88=90=E3=82=92=20Unity?= =?UTF-8?q?=20=E3=82=B9=E3=83=AC=E3=83=83=E3=83=89=E3=81=A7=E8=A1=8C?= =?UTF-8?q?=E3=81=86=E3=82=88=E3=81=86=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sora.cpp | 49 +++++++++++++++++++++++++++++++++++++++---------- src/sora.h | 6 +++++- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/src/sora.cpp b/src/sora.cpp index d14ca65..8f5518b 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -541,11 +541,10 @@ void Sora::Disconnect() { void Sora::SwitchCamera(const sora_conf::internal::CameraConfig& cc) { RTC_LOG(LS_INFO) << "SwitchCamera: " << jsonif::to_json(cc); - boost::asio::post(*ioc_, [self = shared_from_this(), cc = cc]() { - self->DoSwitchCamera(cc); - }); -} -void Sora::DoSwitchCamera(const sora_conf::internal::CameraConfig& cc) { + + // このあたりのキャプチャラ作成は、IO スレッドではなく Unity スレッドで行う。 + // （DoConnect で作成したキャプチャラは Unity スレッド上で実行されてるので、 + // 全て同じスレッドでやる必要がある） std::function on_frame; if (on_capturer_frame_) { on_frame = [on_frame = @@ -582,10 +581,40 @@ void Sora::DoSwitchCamera(const sora_conf::internal::CameraConfig& cc) { std::string video_track_id = rtc::CreateRandomString(16); auto video_track = factory_->CreateVideoTrack(video_track_id, capturer.get()); - renderer_->ReplaceTrack(video_track_.get(), video_track.get()); - video_sender_->SetTrack(video_track.get()); + + // video_track_ をこのスレッド(Unity スレッド)で設定したいので、 + // IO スレッドの実行完了を待つ + std::promise p; + std::future f = p.get_future(); + boost::asio::post(*ioc_, + [self = shared_from_this(), cc = cc, video_track, &p]() { + self->DoSwitchCamera(cc, video_track); + p.set_value(); + }); + f.wait(); + video_track_ = video_track; } +void Sora::DoSwitchCamera( + const sora_conf::internal::CameraConfig& cc, + rtc::scoped_refptr video_track) { + if (video_track_ == nullptr) { + webrtc::RTCErrorOr> + video_result = signaling_->GetPeerConnection()->AddTrack(video_track, + {stream_id_}); + video_sender_ = video_result.value(); + + auto track_id = renderer_->AddTrack(video_track.get()); + PushEvent([this, track_id]() { + if (on_add_track_) { + on_add_track_(track_id, ""); + } + }); + } else { + renderer_->ReplaceTrack(video_track_.get(), video_track.get()); + } + video_sender_->SetTrack(video_track.get()); +} void Sora::RenderCallbackStatic(int event_id) { auto sora = (Sora*)IdPointer::Instance().Lookup(event_id); @@ -838,16 +867,16 @@ void Sora::OnSetOffer(std::string offer) { on_set_offer_(offer); } }); - std::string stream_id = rtc::CreateRandomString(16); + stream_id_ = rtc::CreateRandomString(16); if (audio_track_ != nullptr) { webrtc::RTCErrorOr> audio_result = signaling_->GetPeerConnection()->AddTrack(audio_track_, - {stream_id}); + {stream_id_}); } if (video_track_ != nullptr) { webrtc::RTCErrorOr> video_result = signaling_->GetPeerConnection()->AddTrack(video_track_, - {stream_id}); + {stream_id_}); video_sender_ = video_result.value(); } diff --git a/src/sora.h b/src/sora.h index 3b6d485..7a72db8 100644 --- a/src/sora.h +++ b/src/sora.h @@ -92,7 +92,9 @@ class Sora : public std::enable_shared_from_this, private: void DoConnect(const sora_conf::internal::ConnectConfig& config, std::function on_disconnect); - void DoSwitchCamera(const sora_conf::internal::CameraConfig& cc); + void DoSwitchCamera( + const sora_conf::internal::CameraConfig& cc, + rtc::scoped_refptr video_track); static rtc::scoped_refptr CreateADM( webrtc::TaskQueueFactory* task_queue_factory, @@ -168,6 +170,8 @@ class Sora : public std::enable_shared_from_this, std::map connection_ids_; + std::string stream_id_; + rtc::scoped_refptr capturer_; int capturer_type_ = 0; std::shared_ptr capturer_sink_; From 2c2d2e95c3dcaeaa30f8ed2d36deeb5d10692286 Mon Sep 17 00:00:00 2001 From: melpon Date: Fri, 8 Sep 2023 04:01:58 +0900 Subject: [PATCH 18/42] =?UTF-8?q?Sora=20C++=20SDK=20=E3=82=92=202023.12.0?= =?UTF-8?q?=20=E3=81=AB=E4=B8=8A=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 4 ++++ VERSION | 4 ++-- src/sora.cpp | 11 +++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c4df10c..eec1496 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,10 @@ - [CHANGE] `Sora.Config` 中にあるキャプチャラに関するフィールドを `Sora.CameraConfig` に移動する - 修正方法は （TODO: @miosakuma がドキュメントへのリンクに差し替える） を参照して下さい - @melpon +- [UPDATE] Sora C++ SDK を `2023.12.0` に上げる + - @melpon +- [UPDATE] libwebrtc を `m116.5845.6.1` に上げる + - @melpon - [ADD] 接続中にキャプチャラを切り替える機能を実装 - @melpon diff --git a/VERSION b/VERSION index 6c5721c..03585f8 100644 --- a/VERSION +++ b/VERSION @@ -1,6 +1,6 @@ SORA_UNITY_SDK_VERSION=2023.3.0 -SORA_CPP_SDK_VERSION=2023.10.0 -WEBRTC_BUILD_VERSION=m115.5790.7.0 +SORA_CPP_SDK_VERSION=2023.12.0 +WEBRTC_BUILD_VERSION=m116.5845.6.1 BOOST_VERSION=1.82.0 LYRA_VERSION=1.3.0 CMAKE_VERSION=3.26.4 diff --git a/src/sora.cpp b/src/sora.cpp index 8f5518b..aba4e1b 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -34,6 +34,7 @@ #endif #ifdef SORA_UNITY_SDK_IOS +#include #include "mac_helper/ios_audio_init.h" #endif @@ -58,6 +59,11 @@ Sora::~Sora() { if (capturer_ != nullptr && capturer_type_ == 0) { static_cast(capturer_.get())->Stop(); } +#endif +#if defined(SORA_UNITY_SDK_IOS) || defined(SORA_UNITY_SDK_MACOS) + if (capturer_ != nullptr && capturer_type_ == 0) { + static_cast(capturer_.get())->Stop(); + } #endif if (capturer_ != nullptr && capturer_type_ != 0) { static_cast(capturer_.get())->Stop(); @@ -561,6 +567,11 @@ void Sora::SwitchCamera(const sora_conf::internal::CameraConfig& cc) { if (capturer_ != nullptr && capturer_type_ == 0) { static_cast(capturer_.get())->Stop(); } +#endif +#if defined(SORA_UNITY_SDK_IOS) || defined(SORA_UNITY_SDK_MACOS) + if (capturer_ != nullptr && capturer_type_ == 0) { + static_cast(capturer_.get())->Stop(); + } #endif if (capturer_ != nullptr && capturer_type_ != 0) { static_cast(capturer_.get())->Stop(); From 75702954f179001dbf21026cc890af2c6ce82cbf Mon Sep 17 00:00:00 2001 From: melpon Date: Fri, 8 Sep 2023 05:24:07 +0900 Subject: [PATCH 19/42] =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sora.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sora.cpp b/src/sora.cpp index aba4e1b..0d55d08 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -38,6 +38,10 @@ #include "mac_helper/ios_audio_init.h" #endif +#ifdef SORA_UNITY_SDK_MACOS +#include +#endif + namespace sora_unity_sdk { Sora::Sora(UnityContext* context) : context_(context) { From 32ecdd96e3c11d7512eea4bb5f42d4e0a53474e6 Mon Sep 17 00:00:00 2001 From: melpon Date: Sun, 10 Sep 2023 03:44:54 +0900 Subject: [PATCH 20/42] =?UTF-8?q?ContextUtils.initialize(context)=20?= =?UTF-8?q?=E3=82=92=E5=91=BC=E3=82=93=E3=81=A7=20Android=20=E3=82=B3?= =?UTF-8?q?=E3=83=B3=E3=83=86=E3=82=AD=E3=82=B9=E3=83=88=E3=82=92=E8=A8=AD?= =?UTF-8?q?=E5=AE=9A=E3=81=97=E3=81=A6=E3=81=8A=E3=81=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/android_helper/android_context.cpp | 8 ++++++++ src/sora.cpp | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/android_helper/android_context.cpp b/src/android_helper/android_context.cpp index 14fb530..6327788 100644 --- a/src/android_helper/android_context.cpp +++ b/src/android_helper/android_context.cpp @@ -23,6 +23,14 @@ webrtc::ScopedJavaLocalRef GetAndroidApplicationContext(JNIEnv* env) { webrtc::ScopedJavaLocalRef context( env, env->CallObjectMethod(activity.obj(), ctxid)); + // org.webrtc.ContextUtils.initialize(context) + // を呼ぶ。これをしないと ADM が生成できない + webrtc::ScopedJavaLocalRef cucls = + webrtc::GetClass(env, "org/webrtc/ContextUtils"); + jmethodID initid = env->GetStaticMethodID(cucls.obj(), "initialize", + "(Landroid/content/Context;)V"); + env->CallStaticObjectMethod(cucls.obj(), initid, context.obj()); + return context; } diff --git a/src/sora.cpp b/src/sora.cpp index e871cde..077089e 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -262,7 +262,8 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, cc.unity_audio_input, cc.unity_audio_output, on_handle_audio_, cc.audio_recording_device, cc.audio_playout_device, dependencies.worker_thread, worker_env, worker_context); - media_dependencies.adm = unity_adm_; + dependencies.worker_thread->BlockingCall( + [&] { media_dependencies.adm = unity_adm_; }); #if defined(SORA_UNITY_SDK_ANDROID) dependencies.worker_thread->BlockingCall([worker_env, worker_context] { From 5f6c46b9b26359ba82e581d86f0f07abedcded85 Mon Sep 17 00:00:00 2001 From: melpon Date: Sun, 10 Sep 2023 13:36:31 +0900 Subject: [PATCH 21/42] =?UTF-8?q?Object=20=E3=81=98=E3=82=83=E3=81=AA?= =?UTF-8?q?=E3=81=8F=E3=81=A6=20Void?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/android_helper/android_context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/android_helper/android_context.cpp b/src/android_helper/android_context.cpp index 6327788..e4f29b2 100644 --- a/src/android_helper/android_context.cpp +++ b/src/android_helper/android_context.cpp @@ -29,7 +29,7 @@ webrtc::ScopedJavaLocalRef GetAndroidApplicationContext(JNIEnv* env) { webrtc::GetClass(env, "org/webrtc/ContextUtils"); jmethodID initid = env->GetStaticMethodID(cucls.obj(), "initialize", "(Landroid/content/Context;)V"); - env->CallStaticObjectMethod(cucls.obj(), initid, context.obj()); + env->CallStaticVoidMethod(cucls.obj(), initid, context.obj()); return context; } From b062623df43a4fc4fb6d4963d6384c98bb1c0423 Mon Sep 17 00:00:00 2001 From: melpon Date: Sun, 10 Sep 2023 14:54:28 +0900 Subject: [PATCH 22/42] =?UTF-8?q?offer=20=E3=81=8C=E6=9D=A5=E3=82=8B?= =?UTF-8?q?=E3=81=BE=E3=81=A7=20SwitchCamera=20=E3=82=92=E5=BC=BE=E3=81=8F?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sora.cpp | 6 ++++++ src/sora.h | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/sora.cpp b/src/sora.cpp index 077089e..3d6ecb5 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -508,6 +508,10 @@ void Sora::Disconnect() { } void Sora::SwitchCamera(const sora_conf::internal::CameraConfig& cc) { + if (!set_offer_) { + return; + } + RTC_LOG(LS_INFO) << "SwitchCamera: " << jsonif::to_json(cc); // このあたりのキャプチャラ作成は、IO スレッドではなく Unity スレッドで行う。 @@ -869,6 +873,8 @@ void Sora::OnSetOffer(std::string offer) { } }); } + + set_offer_ = true; } void Sora::OnDisconnect(sora::SoraSignalingErrorCode ec, std::string message) { RTC_LOG(LS_INFO) << "OnDisconnect: " << message; diff --git a/src/sora.h b/src/sora.h index 60898e1..6c4b3d1 100644 --- a/src/sora.h +++ b/src/sora.h @@ -183,6 +183,8 @@ class Sora : public std::enable_shared_from_this, #if defined(SORA_UNITY_SDK_ANDROID) webrtc::ScopedJavaGlobalRef android_context_; #endif + + std::atomic set_offer_; }; } // namespace sora_unity_sdk From cabef4ffc805fa269d7b08a44c16b47695d24a28 Mon Sep 17 00:00:00 2001 From: melpon Date: Sun, 10 Sep 2023 14:58:44 +0900 Subject: [PATCH 23/42] =?UTF-8?q?=E5=88=9D=E6=9C=9F=E5=80=A4=E3=82=92?= =?UTF-8?q?=E8=A8=AD=E5=AE=9A=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sora.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sora.h b/src/sora.h index 6c4b3d1..ebbbd33 100644 --- a/src/sora.h +++ b/src/sora.h @@ -184,7 +184,7 @@ class Sora : public std::enable_shared_from_this, webrtc::ScopedJavaGlobalRef android_context_; #endif - std::atomic set_offer_; + std::atomic set_offer_ = false; }; } // namespace sora_unity_sdk From 081d8aa0bdfaf94957fa3013ade74f0c11d29ffc Mon Sep 17 00:00:00 2001 From: melpon Date: Sun, 10 Sep 2023 15:59:06 +0900 Subject: [PATCH 24/42] =?UTF-8?q?Sora=20C++=20SDK=20=E3=82=92=202023.12.1?= =?UTF-8?q?=20=E3=81=AB=E4=B8=8A=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4fdf94e..cb0096d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,7 +16,7 @@ - @melpon - [UPDATE] SoraClientContext を利用してコードを短くする - @melpon -- [UPDATE] Sora C++ SDK を `2023.12.0` に上げる +- [UPDATE] Sora C++ SDK を `2023.12.1` に上げる - @melpon - [UPDATE] libwebrtc を `m116.5845.6.1` に上げる - @melpon diff --git a/VERSION b/VERSION index 03585f8..1a9b4c2 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ SORA_UNITY_SDK_VERSION=2023.3.0 -SORA_CPP_SDK_VERSION=2023.12.0 +SORA_CPP_SDK_VERSION=2023.12.1 WEBRTC_BUILD_VERSION=m116.5845.6.1 BOOST_VERSION=1.82.0 LYRA_VERSION=1.3.0 From 60d0b1dcd7e4d16b0a1a4b0240ba4a3798fee73b Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 12 Sep 2023 13:45:11 +0900 Subject: [PATCH 25/42] =?UTF-8?q?AudioOutputHelper=20=E3=81=AE=E3=82=A8?= =?UTF-8?q?=E3=82=B9=E3=83=91=E3=83=BC=E5=AE=9F=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sora/Sora.cs | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/unity.cpp | 35 ++++++++++++++++++++++++++++++++++ src/unity.h | 11 +++++++++++ 3 files changed, 99 insertions(+) diff --git a/Sora/Sora.cs b/Sora/Sora.cs index 16659be..d32cd22 100644 --- a/Sora/Sora.cs +++ b/Sora/Sora.cs @@ -926,4 +926,57 @@ public string ConnectedSignalingURL private static extern void sora_get_selected_signaling_url(IntPtr p, [Out] byte[] buf, int size); [DllImport(DllName)] private static extern void sora_get_connected_signaling_url(IntPtr p, [Out] byte[] buf, int size); + + class AudioOutputHelper : IDisposable + { + private delegate void ChangeRouteCallbackDelegate(IntPtr userdata); + + [AOT.MonoPInvokeCallback(typeof(ChangeRouteCallbackDelegate))] + static private void ChangeRouteCallback(IntPtr userdata) + { + var callback = GCHandle.FromIntPtr(userdata).Target as Action; + callback(); + } + + GCHandle onChangeRouteHandle; + IntPtr p; + + public AudioOutputHelper(Action onChangeRoute) + { + onChangeRouteHandle = GCHandle.Alloc(onChangeRoute); + p = sora_audio_output_helper_create(ChangeRouteCallback, GCHandle.ToIntPtr(onChangeRouteHandle)); + } + + public void Dispose() + { + sora_audio_output_helper_destroy(p); + onChangeRouteHandle.Free(); + } + + public bool IsHandsfree() + { + return sora_audio_output_helper_is_handsfree(p) != 0; + } + + public void SetHandsfree(bool enabled) + { + sora_audio_output_helper_set_handsfree(p, enabled ? 1 : 0); + } + +#if UNITY_IOS && !UNITY_EDITOR + private const string DllName = "__Internal"; +#else + private const string DllName = "SoraUnitySdk"; +#endif + + [DllImport(DllName)] + private static extern IntPtr sora_audio_output_helper_create(change_route_cb_t cb, IntPtr userdata); + [DllImport(DllName)] + private static extern void sora_audio_output_helper_destroy(IntPtr p); + [DllImport(DllName)] + private static extern int sora_audio_output_helper_is_handsfree(IntPtr p); + [DllImport(DllName)] + private static extern void sora_audio_output_helper_set_handsfree(IntPtr p, int enabled); + } + } diff --git a/src/unity.cpp b/src/unity.cpp index 49e426c..09d63c1 100644 --- a/src/unity.cpp +++ b/src/unity.cpp @@ -1,4 +1,8 @@ #include "unity.h" + +// Sora +#include + #include "device_list.h" #include "sora.h" #include "sora_conf.json.h" @@ -274,6 +278,37 @@ void sora_get_connected_signaling_url(void* p, void* buf, int size) { std::memcpy(buf, str.c_str(), std::min(size, (int)str.size())); } +struct AudioOutputHelperImpl : public sora::AudioOutputHelper { + AudioOutputHelperImpl(change_route_cb_t cb, void* userdata) + : helper_(sora::CreateAudioOutputHelper()), + cb_(cb), + userdata_(userdata) {} + void OnChangeRoute() override { cb_(userdata_); } + + bool IsHandsFree() override { return helper_->IsHandsFree(); } + void SetHandsFree(bool enabled) override { helper_->SetHandsFree(enabled); } + + private: + std::unique_ptr helper_; + change_route_cb_t cb_; + void* userdata_; +}; + +void* sora_audio_output_helper_create(change_route_cb_t cb, void* userdata) { + return new AudioOutputHelperImpl(cb, userdata); +} +void sora_audio_output_helper_destroy(void* p) { + delete (AudioOutputHelperImpl*)p; +} +unity_bool_t sora_audio_output_helper_is_handsfree(void* p) { + auto helper = (AudioOutputHelperImpl*)p; + return helper->IsHandsFree() ? 1 : 0; +} +void sora_audio_output_helper_set_handsfree(void* p, unity_bool_t enabled) { + auto helper = (AudioOutputHelperImpl*)p; + helper->SetHandsFree(enabled != 0); +} + // iOS の場合は static link で名前が被る可能性があるので、別の名前にしておく void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API #if defined(SORA_UNITY_SDK_IOS) diff --git a/src/unity.h b/src/unity.h index d7447ee..336084c 100644 --- a/src/unity.h +++ b/src/unity.h @@ -116,6 +116,17 @@ UNITY_INTERFACE_EXPORT void sora_get_connected_signaling_url(void* p, void* buf, int size); +typedef void (*change_route_cb_t)(void* userdata); +UNITY_INTERFACE_EXPORT void* sora_audio_output_helper_create( + change_route_cb_t cb, + void* userdata); +UNITY_INTERFACE_EXPORT void sora_audio_output_helper_destroy(void* p); +UNITY_INTERFACE_EXPORT unity_bool_t +sora_audio_output_helper_is_handsfree(void* p); +UNITY_INTERFACE_EXPORT void sora_audio_output_helper_set_handsfree( + void* p, + unity_bool_t enabled); + #ifdef __cplusplus } #endif From ca5c67c0189661881d84ebde78a8d69e6fb3aca3 Mon Sep 17 00:00:00 2001 From: miosakuma Date: Tue, 12 Sep 2023 17:08:27 +0900 Subject: [PATCH 26/42] =?UTF-8?q?Sora=20C++=20SDK=20=E3=82=92=202023.13.0?= =?UTF-8?q?=20=E3=81=AB=E4=B8=8A=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1a9b4c2..94e1eb5 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ SORA_UNITY_SDK_VERSION=2023.3.0 -SORA_CPP_SDK_VERSION=2023.12.1 +SORA_CPP_SDK_VERSION=2023.13.0 WEBRTC_BUILD_VERSION=m116.5845.6.1 BOOST_VERSION=1.82.0 LYRA_VERSION=1.3.0 From 7a5b97f7cfc6d7c02713f58b4d8fef91d2c94f56 Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 12 Sep 2023 18:32:08 +0900 Subject: [PATCH 27/42] =?UTF-8?q?=E3=82=B3=E3=83=B3=E3=83=91=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E3=82=A8=E3=83=A9=E3=83=BC=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sora/Sora.cs | 2 +- src/unity.cpp | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Sora/Sora.cs b/Sora/Sora.cs index d32cd22..5a4256f 100644 --- a/Sora/Sora.cs +++ b/Sora/Sora.cs @@ -970,7 +970,7 @@ public void SetHandsfree(bool enabled) #endif [DllImport(DllName)] - private static extern IntPtr sora_audio_output_helper_create(change_route_cb_t cb, IntPtr userdata); + private static extern IntPtr sora_audio_output_helper_create(ChangeRouteCallbackDelegate cb, IntPtr userdata); [DllImport(DllName)] private static extern void sora_audio_output_helper_destroy(IntPtr p); [DllImport(DllName)] diff --git a/src/unity.cpp b/src/unity.cpp index 09d63c1..1a2ea7c 100644 --- a/src/unity.cpp +++ b/src/unity.cpp @@ -278,18 +278,19 @@ void sora_get_connected_signaling_url(void* p, void* buf, int size) { std::memcpy(buf, str.c_str(), std::min(size, (int)str.size())); } -struct AudioOutputHelperImpl : public sora::AudioOutputHelper { +struct AudioOutputHelperImpl : public sora::AudioOutputHelperInterface, + public sora::AudioChangeRouteObserver { AudioOutputHelperImpl(change_route_cb_t cb, void* userdata) - : helper_(sora::CreateAudioOutputHelper()), + : helper_(sora::CreateAudioOutputHelper(this)), cb_(cb), userdata_(userdata) {} void OnChangeRoute() override { cb_(userdata_); } - bool IsHandsFree() override { return helper_->IsHandsFree(); } - void SetHandsFree(bool enabled) override { helper_->SetHandsFree(enabled); } + bool IsHandsfree() override { return helper_->IsHandsfree(); } + void SetHandsfree(bool enabled) override { helper_->SetHandsfree(enabled); } private: - std::unique_ptr helper_; + std::unique_ptr helper_; change_route_cb_t cb_; void* userdata_; }; @@ -302,11 +303,11 @@ void sora_audio_output_helper_destroy(void* p) { } unity_bool_t sora_audio_output_helper_is_handsfree(void* p) { auto helper = (AudioOutputHelperImpl*)p; - return helper->IsHandsFree() ? 1 : 0; + return helper->IsHandsfree() ? 1 : 0; } void sora_audio_output_helper_set_handsfree(void* p, unity_bool_t enabled) { auto helper = (AudioOutputHelperImpl*)p; - helper->SetHandsFree(enabled != 0); + helper->SetHandsfree(enabled != 0); } // iOS の場合は static link で名前が被る可能性があるので、別の名前にしておく From 655d10ae8b2d4f663a8e1fc36abb9bc88e36c420 Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 12 Sep 2023 18:37:14 +0900 Subject: [PATCH 28/42] =?UTF-8?q?=E3=82=A4=E3=83=B3=E3=82=BF=E3=83=BC?= =?UTF-8?q?=E3=83=95=E3=82=A7=E3=83=BC=E3=82=B9=E3=81=AF=E3=81=84=E3=82=89?= =?UTF-8?q?=E3=81=AA=E3=81=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/unity.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/unity.cpp b/src/unity.cpp index 1a2ea7c..2695a4b 100644 --- a/src/unity.cpp +++ b/src/unity.cpp @@ -278,16 +278,15 @@ void sora_get_connected_signaling_url(void* p, void* buf, int size) { std::memcpy(buf, str.c_str(), std::min(size, (int)str.size())); } -struct AudioOutputHelperImpl : public sora::AudioOutputHelperInterface, - public sora::AudioChangeRouteObserver { +struct AudioOutputHelperImpl : public sora::AudioChangeRouteObserver { AudioOutputHelperImpl(change_route_cb_t cb, void* userdata) : helper_(sora::CreateAudioOutputHelper(this)), cb_(cb), userdata_(userdata) {} void OnChangeRoute() override { cb_(userdata_); } - bool IsHandsfree() override { return helper_->IsHandsfree(); } - void SetHandsfree(bool enabled) override { helper_->SetHandsfree(enabled); } + bool IsHandsfree() { return helper_->IsHandsfree(); } + void SetHandsfree(bool enabled) { helper_->SetHandsfree(enabled); } private: std::unique_ptr helper_; From 9b1a8e4284d298b93b2c66b0182f5b2d3d3b9eb2 Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 12 Sep 2023 20:00:33 +0900 Subject: [PATCH 29/42] =?UTF-8?q?Sora=20C++=20SDK=20=E3=82=92=202023.13.1?= =?UTF-8?q?=20=E3=81=AB=E4=B8=8A=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cb0096d..09cbf6c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,7 +16,7 @@ - @melpon - [UPDATE] SoraClientContext を利用してコードを短くする - @melpon -- [UPDATE] Sora C++ SDK を `2023.12.1` に上げる +- [UPDATE] Sora C++ SDK を `2023.13.1` に上げる - @melpon - [UPDATE] libwebrtc を `m116.5845.6.1` に上げる - @melpon diff --git a/VERSION b/VERSION index 94e1eb5..a1de181 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ SORA_UNITY_SDK_VERSION=2023.3.0 -SORA_CPP_SDK_VERSION=2023.13.0 +SORA_CPP_SDK_VERSION=2023.13.1 WEBRTC_BUILD_VERSION=m116.5845.6.1 BOOST_VERSION=1.82.0 LYRA_VERSION=1.3.0 From 8267025270405e6ec6854ef837ce70be618994ad Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 12 Sep 2023 22:43:39 +0900 Subject: [PATCH 30/42] =?UTF-8?q?public=20=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sora/Sora.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sora/Sora.cs b/Sora/Sora.cs index 5a4256f..132625f 100644 --- a/Sora/Sora.cs +++ b/Sora/Sora.cs @@ -927,7 +927,7 @@ public string ConnectedSignalingURL [DllImport(DllName)] private static extern void sora_get_connected_signaling_url(IntPtr p, [Out] byte[] buf, int size); - class AudioOutputHelper : IDisposable + public class AudioOutputHelper : IDisposable { private delegate void ChangeRouteCallbackDelegate(IntPtr userdata); From d9edce7e1f2b4b55947de9dac335d34bdb175ff1 Mon Sep 17 00:00:00 2001 From: miosakuma Date: Wed, 20 Sep 2023 10:55:30 +0900 Subject: [PATCH 31/42] =?UTF-8?q?Github=20Actions=20=E3=81=AE=20actions/ch?= =?UTF-8?q?eckout=20=E3=82=92=20v4=20=E3=81=AB=E3=81=82=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 913432d..4192600 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ jobs: - windows_x86_64 runs-on: windows-2019 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Cache id: cache uses: actions/cache@v3 @@ -49,7 +49,7 @@ jobs: - ios runs-on: macos-12 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Cache id: cache uses: actions/cache@v3 @@ -97,7 +97,7 @@ jobs: - ubuntu-20.04_x86_64 runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Cache id: cache uses: actions/cache@v3 @@ -143,7 +143,7 @@ jobs: - build-macos - build-ubuntu steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Download artifacts uses: actions/download-artifact@v3 - name: Packaging @@ -184,7 +184,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download SoraUnitySdk uses: actions/download-artifact@v3 with: From 4c0b4e05cae6a26a3275e5b767d3fcf3c1ff3f9f Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 20 Sep 2023 19:31:06 +0900 Subject: [PATCH 32/42] =?UTF-8?q?IosAudioInit=20=E3=82=92=E7=A2=BA?= =?UTF-8?q?=E5=AE=9F=E3=81=AB1=E5=9B=9E=E3=81=97=E3=81=8B=E5=91=BC?= =?UTF-8?q?=E3=81=B0=E3=82=8C=E3=81=AA=E3=81=84=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sora.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sora.cpp b/src/sora.cpp index 3d6ecb5..d32edfc 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -206,8 +206,8 @@ void Sora::Connect(const sora_conf::internal::ConnectConfig& cc) { on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, "Failed to IosAudioInit: error=" + error); } - ios_audio_init = true; }); + ios_audio_init = true; } DoConnect(cc, std::move(on_disconnect)); #else From 9cc4ed804dc7ae25d65b2ffa49600b202bc6459d Mon Sep 17 00:00:00 2001 From: torikizi Date: Mon, 2 Oct 2023 15:10:41 +0900 Subject: [PATCH 33/42] =?UTF-8?q?Sora=20C++=20SDK=20=E3=82=92=202023.14.0?= =?UTF-8?q?=20=E3=81=AB=E3=82=A2=E3=83=83=E3=83=97=E3=83=87=E3=83=BC?= =?UTF-8?q?=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index a1de181..4acb7ff 100644 --- a/VERSION +++ b/VERSION @@ -1,7 +1,7 @@ SORA_UNITY_SDK_VERSION=2023.3.0 -SORA_CPP_SDK_VERSION=2023.13.1 -WEBRTC_BUILD_VERSION=m116.5845.6.1 -BOOST_VERSION=1.82.0 +SORA_CPP_SDK_VERSION=2023.14.0 +WEBRTC_BUILD_VERSION=m117.5938.2.0 +BOOST_VERSION=1.83.0 LYRA_VERSION=1.3.0 CMAKE_VERSION=3.26.4 PROTOBUF_VERSION=21.1 From b40e8aff5c1e6b38634e289393d7d63411cba8e3 Mon Sep 17 00:00:00 2001 From: melpon Date: Mon, 16 Oct 2023 02:20:45 +0900 Subject: [PATCH 34/42] =?UTF-8?q?IosAudioInit=20=E3=82=92=E5=88=9D?= =?UTF-8?q?=E5=9B=9E=E3=81=AE=E3=81=BF=E5=AE=9F=E8=A1=8C=E3=81=99=E3=82=8B?= =?UTF-8?q?=E3=81=AE=E3=81=A7=E3=81=AF=E3=81=AA=E3=81=8F=E3=80=81=E5=A4=9A?= =?UTF-8?q?=E9=87=8D=E3=81=AB=E5=AE=9F=E8=A1=8C=E3=81=95=E3=82=8C=E3=82=8B?= =?UTF-8?q?=E3=81=AE=E3=81=A0=E3=81=91=E9=98=B2=E3=81=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sora.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/sora.cpp b/src/sora.cpp index d32edfc..4eb1f3b 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -197,8 +197,9 @@ void Sora::Connect(const sora_conf::internal::ConnectConfig& cc) { return; } - static bool ios_audio_init = false; - if (!ios_audio_init) { + static bool ios_audio_initializing = false; + if (!ios_audio_initializing) { + ios_audio_initializing = true; IosAudioInit( [this, on_disconnect = std::move(on_disconnect)](std::string error) { if (!error.empty()) { @@ -206,8 +207,8 @@ void Sora::Connect(const sora_conf::internal::ConnectConfig& cc) { on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, "Failed to IosAudioInit: error=" + error); } + ios_audio_initializing = false; }); - ios_audio_init = true; } DoConnect(cc, std::move(on_disconnect)); #else From 559e69980af3703fea256df121a493dbe70895b9 Mon Sep 17 00:00:00 2001 From: melpon Date: Mon, 16 Oct 2023 02:28:00 +0900 Subject: [PATCH 35/42] =?UTF-8?q?AudioOutputHelper.Dispose()=20=E3=82=92?= =?UTF-8?q?=E8=A4=87=E6=95=B0=E5=9B=9E=E5=91=BC=E3=82=93=E3=81=A7=E3=82=82?= =?UTF-8?q?=E3=82=AF=E3=83=A9=E3=83=83=E3=82=B7=E3=83=A5=E3=81=97=E3=81=AA?= =?UTF-8?q?=E3=81=84=E3=82=88=E3=81=86=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sora/Sora.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Sora/Sora.cs b/Sora/Sora.cs index 132625f..26610fd 100644 --- a/Sora/Sora.cs +++ b/Sora/Sora.cs @@ -949,8 +949,12 @@ public AudioOutputHelper(Action onChangeRoute) public void Dispose() { - sora_audio_output_helper_destroy(p); - onChangeRouteHandle.Free(); + if (p != IntPtr.Zero) + { + sora_audio_output_helper_destroy(p); + onChangeRouteHandle.Free(); + p = IntPtr.Zero; + } } public bool IsHandsfree() From daa0d72fd70c8b801df6d2ca1af672e134019948 Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 18 Oct 2023 15:13:56 +0900 Subject: [PATCH 36/42] =?UTF-8?q?=E7=8F=BE=E5=9C=A8=E3=81=AE=20AVAudioSess?= =?UTF-8?q?ion=20=E3=81=AE=E5=80=A4=E3=82=92=20RTCAudioSessionConfiguratio?= =?UTF-8?q?n=20=E3=81=AB=E5=8F=8D=E6=98=A0=E3=81=95=E3=81=9B=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/mac_helper/ios_audio_init.h | 4 +++- src/mac_helper/ios_audio_init.mm | 7 +++++++ src/sora.cpp | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/mac_helper/ios_audio_init.h b/src/mac_helper/ios_audio_init.h index 4afadfc..84a5970 100644 --- a/src/mac_helper/ios_audio_init.h +++ b/src/mac_helper/ios_audio_init.h @@ -7,7 +7,9 @@ namespace sora_unity_sdk { void IosAudioInit(std::function on_complete); +// 現在の AVAudioSession の設定を RTCAudioSessionConfiguration に反映させる +void IosSyncAudioSession(); -} +} // namespace sora_unity_sdk #endif diff --git a/src/mac_helper/ios_audio_init.mm b/src/mac_helper/ios_audio_init.mm index ce0ffe7..e54ece5 100644 --- a/src/mac_helper/ios_audio_init.mm +++ b/src/mac_helper/ios_audio_init.mm @@ -19,4 +19,11 @@ void IosAudioInit(std::function on_complete) { }]; } +void IosSyncAudioSession() { + auto config = [RTCAudioSessionConfiguration webRTCConfiguration]; + auto session = [AVAudioSession sharedInstance]; + config.category = session.category; + config.mode = session.mode; +} + } diff --git a/src/sora.cpp b/src/sora.cpp index 4eb1f3b..02c8511 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -197,6 +197,7 @@ void Sora::Connect(const sora_conf::internal::ConnectConfig& cc) { return; } + IosSyncAudioSession(); static bool ios_audio_initializing = false; if (!ios_audio_initializing) { ios_audio_initializing = true; From 34a25ab6a295fc0db52df98de49eb8bd89650ebb Mon Sep 17 00:00:00 2001 From: torikizi Date: Thu, 19 Oct 2023 12:13:49 +0900 Subject: [PATCH 37/42] =?UTF-8?q?Revert=20"=E7=8F=BE=E5=9C=A8=E3=81=AE=20A?= =?UTF-8?q?VAudioSession=20=E3=81=AE=E5=80=A4=E3=82=92=20RTCAudioSessionCo?= =?UTF-8?q?nfiguration=20=E3=81=AB=E5=8F=8D=E6=98=A0=E3=81=95=E3=81=9B?= =?UTF-8?q?=E3=82=8B"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit daa0d72fd70c8b801df6d2ca1af672e134019948. --- src/mac_helper/ios_audio_init.h | 4 +--- src/mac_helper/ios_audio_init.mm | 7 ------- src/sora.cpp | 1 - 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/mac_helper/ios_audio_init.h b/src/mac_helper/ios_audio_init.h index 84a5970..4afadfc 100644 --- a/src/mac_helper/ios_audio_init.h +++ b/src/mac_helper/ios_audio_init.h @@ -7,9 +7,7 @@ namespace sora_unity_sdk { void IosAudioInit(std::function on_complete); -// 現在の AVAudioSession の設定を RTCAudioSessionConfiguration に反映させる -void IosSyncAudioSession(); -} // namespace sora_unity_sdk +} #endif diff --git a/src/mac_helper/ios_audio_init.mm b/src/mac_helper/ios_audio_init.mm index e54ece5..ce0ffe7 100644 --- a/src/mac_helper/ios_audio_init.mm +++ b/src/mac_helper/ios_audio_init.mm @@ -19,11 +19,4 @@ void IosAudioInit(std::function on_complete) { }]; } -void IosSyncAudioSession() { - auto config = [RTCAudioSessionConfiguration webRTCConfiguration]; - auto session = [AVAudioSession sharedInstance]; - config.category = session.category; - config.mode = session.mode; -} - } diff --git a/src/sora.cpp b/src/sora.cpp index 02c8511..4eb1f3b 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -197,7 +197,6 @@ void Sora::Connect(const sora_conf::internal::ConnectConfig& cc) { return; } - IosSyncAudioSession(); static bool ios_audio_initializing = false; if (!ios_audio_initializing) { ios_audio_initializing = true; From b0b1e6141d822eee2e614ceeac52117d72aa7604 Mon Sep 17 00:00:00 2001 From: torikizi Date: Mon, 23 Oct 2023 16:57:02 +0900 Subject: [PATCH 38/42] =?UTF-8?q?CHANGES=20=E3=82=92=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 09cbf6c..4774fda 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,16 +12,16 @@ ## develop - [CHANGE] `Sora.Config` 中にあるキャプチャラに関するフィールドを `Sora.CameraConfig` に移動する - - 修正方法は （TODO: @miosakuma がドキュメントへのリンクに差し替える） を参照して下さい - - @melpon + - 修正方法は （TODO: @miosakuma がドキュメントへのリンクに差し替える） を参照して下さい + - @melpon - [UPDATE] SoraClientContext を利用してコードを短くする - @melpon - [UPDATE] Sora C++ SDK を `2023.13.1` に上げる - - @melpon + - @melpon - [UPDATE] libwebrtc を `m116.5845.6.1` に上げる - - @melpon + - @melpon - [ADD] 接続中にキャプチャラを切り替える機能を実装 - - @melpon + - @melpon - [ADD] デバイスを掴まないようにする `NoVideoDevice`, `NoAudioDevice` を追加 - @melpon - [ADD] ハードウェアエンコーダを利用するかどうかを設定する `UseHardwareEncoder` を追加 @@ -30,6 +30,8 @@ - @melpon - [FIX] IosAudioInit を初回接続の場合のみ呼び出すようにすることで、iOS で連続して接続しようとすると落ちることがあったのを修正 - @melpon +- [FIX] AudioOutputHelper.Dispose() を複数回呼んでもクラッシュしないように修正 + - @melpon ## 2023.3.0 (2023-08-08) @@ -41,13 +43,13 @@ ## 2023.2.0 (2023-07-19) - [UPDATE] libwebrtc を `m114.5735.2.0` に上げる - - @torikizi @miosakuma + - @torikizi @miosakuma - [UPDATE] Sora C++ SDK を `2023.7.2` に上げる - @torikizi @miosakuma - [ADD] ForwardingFilter 機能を使えるようにする - @melpon - [ADD] CodecParams 機能を使えるようにする - - @torikizi + - @torikizi - [FIX] GetStats でデータレースによりエラーが発生するケースについて修正する - @melpon From d9c790ee72c20f390896ee35e0c82709124daf3f Mon Sep 17 00:00:00 2001 From: miosakuma Date: Wed, 25 Oct 2023 15:36:59 +0900 Subject: [PATCH 39/42] =?UTF-8?q?CHANGES=20=E3=82=92=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 4774fda..2e39375 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,7 +12,7 @@ ## develop - [CHANGE] `Sora.Config` 中にあるキャプチャラに関するフィールドを `Sora.CameraConfig` に移動する - - 修正方法は （TODO: @miosakuma がドキュメントへのリンクに差し替える） を参照して下さい + - 修正方法は [Sora Unity SDK ドキュメント](https://sora-unity-sdk.shiguredo.jp/2023_3_0_to_2023_4_0) を参照して下さい - @melpon - [UPDATE] SoraClientContext を利用してコードを短くする - @melpon From 5c564c9a0770e9971b0a8595301ec56e2148a25b Mon Sep 17 00:00:00 2001 From: miosakuma Date: Thu, 26 Oct 2023 17:21:29 +0900 Subject: [PATCH 40/42] =?UTF-8?q?=E5=A4=89=E6=9B=B4=E5=B1=A5=E6=AD=B4?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2e39375..729ad51 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,10 +16,10 @@ - @melpon - [UPDATE] SoraClientContext を利用してコードを短くする - @melpon -- [UPDATE] Sora C++ SDK を `2023.13.1` に上げる - - @melpon -- [UPDATE] libwebrtc を `m116.5845.6.1` に上げる - - @melpon +- [UPDATE] Sora C++ SDK を `2023.14.0` に上げる + - @melpon @torikizi +- [UPDATE] libwebrtc を `m117.5938.2.0` に上げる + - @melpon @torikizi - [ADD] 接続中にキャプチャラを切り替える機能を実装 - @melpon - [ADD] デバイスを掴まないようにする `NoVideoDevice`, `NoAudioDevice` を追加 diff --git a/README.md b/README.md index 5b82c7b..9e07557 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Sora Unity SDK -[![libwebrtc](https://img.shields.io/badge/libwebrtc-115.5790-blue.svg)](https://chromium.googlesource.com/external/webrtc/+/branch-heads/5790) +[![libwebrtc](https://img.shields.io/badge/libwebrtc-117.5938-blue.svg)](https://chromium.googlesource.com/external/webrtc/+/branch-heads/5938) [![GitHub tag](https://img.shields.io/github/tag/shiguredo/sora-unity-sdk.svg)](https://github.com/shiguredo/sora-unity-sdk) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Actions Status](https://github.com/shiguredo/sora-unity-sdk/workflows/build/badge.svg)](https://github.com/shiguredo/sora-unity-sdk/actions) From 97471eb58b663634f7f1779cceee99dbc77a3c9b Mon Sep 17 00:00:00 2001 From: miosakuma Date: Thu, 26 Oct 2023 17:31:09 +0900 Subject: [PATCH 41/42] =?UTF-8?q?README.md=20=E3=82=92=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9e07557..c8b2afa 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Please read https://github.com/shiguredo/oss/blob/master/README.en.md before use ## 対応 Unity バージョン -- Unity 2021.3 (LTS) +- Unity 2022.3 (LTS) ## システム条件 From fb07f5f8e88814fbd73adb532478340362f06087 Mon Sep 17 00:00:00 2001 From: miosakuma Date: Thu, 26 Oct 2023 17:34:25 +0900 Subject: [PATCH 42/42] =?UTF-8?q?2023.4.0=20=E3=83=AA=E3=83=AA=E3=83=BC?= =?UTF-8?q?=E3=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 ++ VERSION | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 729ad51..093f7d3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,8 @@ ## develop +## 2023.4.0 (2023-10-26) + - [CHANGE] `Sora.Config` 中にあるキャプチャラに関するフィールドを `Sora.CameraConfig` に移動する - 修正方法は [Sora Unity SDK ドキュメント](https://sora-unity-sdk.shiguredo.jp/2023_3_0_to_2023_4_0) を参照して下さい - @melpon diff --git a/VERSION b/VERSION index 4acb7ff..3770c2a 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ -SORA_UNITY_SDK_VERSION=2023.3.0 +SORA_UNITY_SDK_VERSION=2023.4.0 SORA_CPP_SDK_VERSION=2023.14.0 WEBRTC_BUILD_VERSION=m117.5938.2.0 BOOST_VERSION=1.83.0