From bb9c1fa42bd55b8657ea5353dd30315aa67191aa Mon Sep 17 00:00:00 2001 From: torikizi Date: Wed, 9 Mar 2022 22:31:45 +0900 Subject: [PATCH 01/70] =?UTF-8?q?FAQ.md=20=E3=81=AE=E4=BF=AE=E6=AD=A3=20-?= =?UTF-8?q?=20=E3=80=8CJetson=20=E3=82=B7=E3=83=AA=E3=83=BC=E3=82=BA?= =?UTF-8?q?=E3=82=92=E4=BD=BF=E3=81=A3=E3=81=A6=20SDL=20=E3=81=A7=20Simulc?= =?UTF-8?q?ast=20=E3=81=AE=20H.264=20=E3=82=92=E5=8F=97=E4=BF=A1=E3=81=99?= =?UTF-8?q?=E3=82=8B=E3=80=8D=E3=81=AB=E3=81=A4=E3=81=84=E3=81=A6=E5=95=8F?= =?UTF-8?q?=E9=A1=8C=E3=81=8C=E8=A7=A3=E6=B6=88=E3=81=95=E3=82=8C=E3=81=9F?= =?UTF-8?q?=E3=81=9F=E3=82=81=E8=A8=98=E8=BC=89=E3=82=92=E5=89=8A=E9=99=A4?= =?UTF-8?q?=E3=81=97=E3=81=BE=E3=81=97=E3=81=9F=E3=80=82=20=20=20=20Releas?= =?UTF-8?q?e=202022.1.0=20=E4=BB=A5=E9=99=8D=E3=81=AF=20Jetson=20=E3=81=AE?= =?UTF-8?q?=E3=83=8F=E3=83=BC=E3=83=89=E3=82=A6=E3=82=A7=E3=82=A2=E3=83=87?= =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=80=E3=83=BC=E3=81=8C=E5=87=BA=E5=8A=9B?= =?UTF-8?q?=E6=99=82=E3=81=AB=E5=87=BA=E5=8A=9B=E3=82=B5=E3=82=A4=E3=82=BA?= =?UTF-8?q?=E3=81=A7=E3=83=95=E3=83=AC=E3=83=BC=E3=83=A0=E3=82=92=E5=88=87?= =?UTF-8?q?=E3=82=8A=E6=8A=9C=E3=81=8F=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=E3=81=95=E3=82=8C=E3=81=9F=E3=81=9F=E3=82=81=E5=BE=93?= =?UTF-8?q?=E6=9D=A5=E3=81=AE=E3=82=88=E3=81=86=E3=81=AB=20H.264=20?= =?UTF-8?q?=E3=81=AE=E8=A7=A3=E5=83=8F=E5=BA=A6=E3=81=AE=E4=BB=95=E6=A7=98?= =?UTF-8?q?=E4=B8=8D=E4=B8=80=E8=87=B4=E3=81=A7=E7=B7=91=E3=81=AE=E7=B7=9A?= =?UTF-8?q?=E3=81=8C=E5=87=BA=E3=82=8B=E3=82=88=E3=81=86=E3=81=AA=E3=81=93?= =?UTF-8?q?=E3=81=A8=E3=81=8C=E3=81=AA=E3=81=8F=E3=81=AA=E3=82=8A=E3=81=BE?= =?UTF-8?q?=E3=81=97=E3=81=9F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/FAQ.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/doc/FAQ.md b/doc/FAQ.md index 2dad965e..2c8b5590 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -120,11 +120,3 @@ Mac (arm64) から FHD でスクリーンキャプチャを配信したい場合 NVIDIA のビデオカードの NVENC を利用することで H.264 が利用可能になります。 ご利用の環境で H.264 が利用できるかどうかは `./momo --video-codec-engines` を使用して H264 の項目をご確認ください。 - -## Jetson シリーズを使って SDL で Simulcast の H.264 を受信する - -Jetson シリーズで Simulcast の H.264 を SDL を使って受信する場合、下部に緑の線が見えることがあります。 - -これは受信している Simulcast の解像度が H.264 の解像度の仕様に不一致な状態で受信しているためで、 Sora の API を使用して受信する解像度を変更してください。 - -API の詳細はこちらの [Sora のドキュメント](https://sora-doc.shiguredo.jp/EXPERIMENTAL_API#adcbc8)をお読みください。 From 4e54f6d73e917e4a877e605656c84e0797c51408 Mon Sep 17 00:00:00 2001 From: tnoho Date: Fri, 11 Mar 2022 01:22:14 +0900 Subject: [PATCH 02/70] =?UTF-8?q?m100=20=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- src/rtc/rtc_connection.cpp | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/VERSION b/VERSION index d5d1ad99..c6d2a036 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ MOMO_VERSION=2022.1.0 -WEBRTC_BUILD_VERSION=99.4844.1.0 +WEBRTC_BUILD_VERSION=100.4896.1.1 BOOST_VERSION=1.78.0 CLI11_VERSION=2.1.2 SDL2_VERSION=2.0.20 diff --git a/src/rtc/rtc_connection.cpp b/src/rtc/rtc_connection.cpp index bf4c37eb..9cb89236 100644 --- a/src/rtc/rtc_connection.cpp +++ b/src/rtc/rtc_connection.cpp @@ -41,8 +41,9 @@ class CreateSessionDescriptionThunk static rtc::scoped_refptr Create( OnSuccessFunc on_success, OnFailureFunc on_failure) { - return new rtc::RefCountedObject( - std::move(on_success), std::move(on_failure)); + return rtc::scoped_refptr( + new rtc::RefCountedObject( + std::move(on_success), std::move(on_failure))); } protected: @@ -81,8 +82,9 @@ class SetSessionDescriptionThunk static rtc::scoped_refptr Create( OnSuccessFunc on_success, OnFailureFunc on_failure) { - return new rtc::RefCountedObject( - std::move(on_success), std::move(on_failure)); + return rtc::scoped_refptr( + new rtc::RefCountedObject( + std::move(on_success), std::move(on_failure))); } protected: @@ -249,7 +251,8 @@ bool RTCConnection::IsVideoEnabled() { rtc::scoped_refptr RTCConnection::GetLocalStream() { - return connection_->local_streams()->at(0); + return rtc::scoped_refptr( + connection_->local_streams()->at(0)); } rtc::scoped_refptr From bfbdc983c2f40727a9b2167395e8f179168fb379 Mon Sep 17 00:00:00 2001 From: tnoho Date: Fri, 11 Mar 2022 01:24:59 +0900 Subject: [PATCH 03/70] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E6=BC=8F=E3=82=8C?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_jetson/jetson_buffer.cpp | 32 ++++++++++++++++-------------- src/rtc/native_buffer.cpp | 3 ++- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/hwenc_jetson/jetson_buffer.cpp b/src/hwenc_jetson/jetson_buffer.cpp index b2a2f759..8fd17f1f 100644 --- a/src/hwenc_jetson/jetson_buffer.cpp +++ b/src/hwenc_jetson/jetson_buffer.cpp @@ -19,15 +19,16 @@ rtc::scoped_refptr JetsonBuffer::Create( int fd, uint32_t pixfmt, std::shared_ptr decoder) { - return new rtc::RefCountedObject( - video_type, - raw_width, - raw_height, - scaled_width, - scaled_height, - fd, - pixfmt, - decoder); + return rtc::scoped_refptr( + new rtc::RefCountedObject( + video_type, + raw_width, + raw_height, + scaled_width, + scaled_height, + fd, + pixfmt, + decoder)); } rtc::scoped_refptr JetsonBuffer::Create( @@ -36,12 +37,13 @@ rtc::scoped_refptr JetsonBuffer::Create( int raw_height, int scaled_width, int scaled_height) { - return new rtc::RefCountedObject( - video_type, - raw_width, - raw_height, - scaled_width, - scaled_height); + return rtc::scoped_refptr( + new rtc::RefCountedObject( + video_type, + raw_width, + raw_height, + scaled_width, + scaled_height)); } webrtc::VideoFrameBuffer::Type JetsonBuffer::type() const { diff --git a/src/rtc/native_buffer.cpp b/src/rtc/native_buffer.cpp index 33fc2a35..0348114a 100644 --- a/src/rtc/native_buffer.cpp +++ b/src/rtc/native_buffer.cpp @@ -17,7 +17,8 @@ int ArgbDataSize(int height, int width) { rtc::scoped_refptr NativeBuffer::Create(webrtc::VideoType video_type, int width, int height) { - return new rtc::RefCountedObject(video_type, width, height); + return rtc::scoped_refptr( + new rtc::RefCountedObject(video_type, width, height)); } webrtc::VideoFrameBuffer::Type NativeBuffer::type() const { From 2e07219e2a23e285d8c1eb8d20f30c3e4aa080da Mon Sep 17 00:00:00 2001 From: voluntas Date: Sat, 12 Mar 2022 22:54:26 +0900 Subject: [PATCH 04/70] =?UTF-8?q?=E5=AD=98=E5=9C=A8=E3=81=97=E3=81=AA?= =?UTF-8?q?=E3=81=84=E3=82=B5=E3=83=B3=E3=83=97=E3=83=AB=E3=82=84=20URL=20?= =?UTF-8?q?=E3=82=92=E6=8C=87=E5=AE=9A=E3=81=97=E3=81=A6=E3=81=84=E3=81=9F?= =?UTF-8?q?=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/USE_SORA.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/doc/USE_SORA.md b/doc/USE_SORA.md index 8f6dc897..4d9423a5 100644 --- a/doc/USE_SORA.md +++ b/doc/USE_SORA.md @@ -4,15 +4,15 @@ Sora は時雨堂が開発、販売している商用 WebRTC SFU 製品です。 https://sora.shiguredo.jp/ -ここでは利用申請することで法人などで無料で検証可能な [Sora Labo](https://sora-labo.shiguredo.jp/) を利用しています。 +ここでは利用申請することで法人などで無料で検証可能な [Sora Labo](https://sora-labo.shiguredo.app/) を利用しています。 Sora Labo の利用申請や使用方法については [Sora Labo のドキュメント](https://github.com/shiguredo/sora-labo-doc)をご確認ください。 ## Sora Labo を使う -GitHub アカウントを用意して https://sora-labo.shiguredo.jp/ にサインアップしてください。 +GitHub アカウントを用意して https://sora-labo.shiguredo.app/ にサインアップしてください。 -### 配信してみる +### 片方向配信をしてみる - チャネル ID に `@<好きな文字列(最大255 バイトまで)>` を指定してください - GitHub ユーザ名が shiguredo とした場合は `shiguredo@sora-labo` のように指定してください @@ -31,12 +31,13 @@ GitHub アカウントを用意して https://sora-labo.shiguredo.jp/ にサイ --channel-id shiguredo@sora-labo \ --video-codec-type VP8 --video-bit-rate 500 \ --audio false \ + --multistream true \ --role sendonly --metadata '{"signaling_key": "xyz"}' ``` -ブラウザでの送受信は Sora Labo にあるサンプルのシングルストリーム受信を利用して確認してください。 +ブラウザでの送受信は Sora Labo にあるサンプルのマルチストリーム受信を利用して確認してください。 -### マルチストリームを使ってみる +### 双方向配信をしてみる GUI 環境で Momo を利用すると、 SDL を利用し音声や映像の受信が可能になります。 @@ -50,7 +51,8 @@ GUI 環境で Momo を利用すると、 SDL を利用し音声や映像の受 --channel-id shiguredo@open-momo \ --video-codec-type VP8 --video-bit-rate 1000 \ --audio false \ - --multistream true --role sendrecv --metadata '{"signaling_key": "xyz"}' + --multistream true \ + --role sendrecv --metadata '{"signaling_key": "xyz"}' ``` ブラウザでの送受信は Sora Labo にあるサンプルのマルチストリーム送受信を利用して確認してください。 @@ -65,12 +67,11 @@ GUI 環境で Momo を利用すると、 SDL を利用し音声や映像の受 wss://..sora.sora-labo.shiguredo.app/signaling \ wss://..sora.sora-labo.shiguredo.app/signaling \ --channel-id shiguredo@sora-labo \ - --simulcast true \ --video-codec-type VP8 --video-bit-rate 500 \ --audio false \ + --multistream true \ + --simulcast true \ --role sendonly --metadata '{"signaling_key": "xyz"}' ``` -ブラウザでの送受信は Sora Labo にあるサンプルのサイマルキャスト受信を利用して確認してください。 - - +ブラウザでの送受信は Sora Labo にあるサンプルのマルチストリームサイマルキャスト受信を利用して確認してください。 From 427efba9c388669c003c420370c8431a6f44b064 Mon Sep 17 00:00:00 2001 From: tnoho Date: Sun, 13 Mar 2022 11:52:46 +0900 Subject: [PATCH 05/70] =?UTF-8?q?Jetson=20=E5=91=A8=E3=82=8A=E4=BB=A5?= =?UTF-8?q?=E5=A4=96=E3=82=82=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_mmal/mmal_buffer.cpp | 3 ++- src/hwenc_nvcodec/nvcodec_h264_encoder.cpp | 12 ++++++------ src/mac_helper/mac_capturer.mm | 5 +++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/hwenc_mmal/mmal_buffer.cpp b/src/hwenc_mmal/mmal_buffer.cpp index e4f9d08c..6d7653fe 100644 --- a/src/hwenc_mmal/mmal_buffer.cpp +++ b/src/hwenc_mmal/mmal_buffer.cpp @@ -8,7 +8,8 @@ rtc::scoped_refptr MMALBuffer::Create(MMAL_BUFFER_HEADER_T* buffer, int width, int height) { - return new rtc::RefCountedObject(buffer, width, height); + return rtc::scoped_refptr( + new rtc::RefCountedObject(buffer, width, height)); } webrtc::VideoFrameBuffer::Type MMALBuffer::type() const { diff --git a/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp b/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp index 0886b9bc..fabe6836 100644 --- a/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp +++ b/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp @@ -44,7 +44,7 @@ NvCodecH264Encoder::NvCodecH264Encoder(const cricket::VideoCodec& codec) char szDesc[80]; size_t result = 0; wcstombs_s(&result, szDesc, adapter_desc.Description, sizeof(szDesc)); - RTC_LOG(INFO) << __FUNCTION__ << "GPU in use: " << szDesc; + RTC_LOG(LS_INFO) << __FUNCTION__ << "GPU in use: " << szDesc; #endif #ifdef __linux__ cuda_.reset(new NvCodecH264EncoderCuda()); @@ -312,11 +312,11 @@ void NvCodecH264Encoder::SetRates( uint32_t new_framerate = (uint32_t)parameters.framerate_fps; uint32_t new_bitrate = parameters.bitrate.get_sum_bps(); - RTC_LOG(INFO) << __FUNCTION__ << " framerate_:" << framerate_ - << " new_framerate: " << new_framerate - << " target_bitrate_bps_:" << target_bitrate_bps_ - << " new_bitrate:" << new_bitrate - << " max_bitrate_bps_:" << max_bitrate_bps_; + RTC_LOG(LS_INFO) << __FUNCTION__ << " framerate_:" << framerate_ + << " new_framerate: " << new_framerate + << " target_bitrate_bps_:" << target_bitrate_bps_ + << " new_bitrate:" << new_bitrate + << " max_bitrate_bps_:" << max_bitrate_bps_; framerate_ = new_framerate; target_bitrate_bps_ = new_bitrate; bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); diff --git a/src/mac_helper/mac_capturer.mm b/src/mac_helper/mac_capturer.mm index 3c5a41cf..d1258491 100644 --- a/src/mac_helper/mac_capturer.mm +++ b/src/mac_helper/mac_capturer.mm @@ -89,8 +89,9 @@ - (void)capturer:(RTCVideoCapturer*)capturer RTC_LOG(LS_ERROR) << "Failed to create MacCapture"; return nullptr; } - return new rtc::RefCountedObject(width, height, target_fps, - device); + return rtc::scoped_refptr( + new rtc::RefCountedObject( + width, height, target_fps, device)); } AVCaptureDevice* MacCapturer::FindVideoDevice( From fff954f78493c2b4108c6cca28c9326a74f7bb30 Mon Sep 17 00:00:00 2001 From: tnoho Date: Sun, 13 Mar 2022 12:15:42 +0900 Subject: [PATCH 06/70] =?UTF-8?q?make=5Fref=5Fcounted=20=E3=81=AB=E5=A4=89?= =?UTF-8?q?=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_jetson/jetson_buffer.cpp | 32 ++++++++++++++---------------- src/hwenc_mmal/mmal_buffer.cpp | 3 +-- src/mac_helper/mac_capturer.mm | 5 ++--- src/main.cpp | 6 ++---- src/rtc/native_buffer.cpp | 3 +-- src/rtc/rtc_connection.cpp | 10 ++++------ 6 files changed, 25 insertions(+), 34 deletions(-) diff --git a/src/hwenc_jetson/jetson_buffer.cpp b/src/hwenc_jetson/jetson_buffer.cpp index 8fd17f1f..c1d184d5 100644 --- a/src/hwenc_jetson/jetson_buffer.cpp +++ b/src/hwenc_jetson/jetson_buffer.cpp @@ -19,16 +19,15 @@ rtc::scoped_refptr JetsonBuffer::Create( int fd, uint32_t pixfmt, std::shared_ptr decoder) { - return rtc::scoped_refptr( - new rtc::RefCountedObject( - video_type, - raw_width, - raw_height, - scaled_width, - scaled_height, - fd, - pixfmt, - decoder)); + return rtc::make_ref_counted( + video_type, + raw_width, + raw_height, + scaled_width, + scaled_height, + fd, + pixfmt, + decoder); } rtc::scoped_refptr JetsonBuffer::Create( @@ -37,13 +36,12 @@ rtc::scoped_refptr JetsonBuffer::Create( int raw_height, int scaled_width, int scaled_height) { - return rtc::scoped_refptr( - new rtc::RefCountedObject( - video_type, - raw_width, - raw_height, - scaled_width, - scaled_height)); + return rtc::make_ref_counted( + video_type, + raw_width, + raw_height, + scaled_width, + scaled_height); } webrtc::VideoFrameBuffer::Type JetsonBuffer::type() const { diff --git a/src/hwenc_mmal/mmal_buffer.cpp b/src/hwenc_mmal/mmal_buffer.cpp index 6d7653fe..af7e0f90 100644 --- a/src/hwenc_mmal/mmal_buffer.cpp +++ b/src/hwenc_mmal/mmal_buffer.cpp @@ -8,8 +8,7 @@ rtc::scoped_refptr MMALBuffer::Create(MMAL_BUFFER_HEADER_T* buffer, int width, int height) { - return rtc::scoped_refptr( - new rtc::RefCountedObject(buffer, width, height)); + return rtc::make_ref_counted(buffer, width, height); } webrtc::VideoFrameBuffer::Type MMALBuffer::type() const { diff --git a/src/mac_helper/mac_capturer.mm b/src/mac_helper/mac_capturer.mm index d1258491..b2ec0ae5 100644 --- a/src/mac_helper/mac_capturer.mm +++ b/src/mac_helper/mac_capturer.mm @@ -89,9 +89,8 @@ - (void)capturer:(RTCVideoCapturer*)capturer RTC_LOG(LS_ERROR) << "Failed to create MacCapture"; return nullptr; } - return rtc::scoped_refptr( - new rtc::RefCountedObject( - width, height, target_fps, device)); + return rtc::make_ref_counted( + width, height, target_fps, device); } AVCaptureDevice* MacCapturer::FindVideoDevice( diff --git a/src/main.cpp b/src/main.cpp index 8abd81ed..567cd5f7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -95,10 +95,8 @@ int main(int argc, char* argv[]) { return nullptr; } auto size = args.GetSize(); - rtc::scoped_refptr capturer( - new rtc::RefCountedObject( - sources[0].id, size.width, size.height, args.framerate)); - return capturer; + return rtc::make_ref_counted( + sources[0].id, size.width, size.height, args.framerate); } #endif diff --git a/src/rtc/native_buffer.cpp b/src/rtc/native_buffer.cpp index 0348114a..07e0d7f4 100644 --- a/src/rtc/native_buffer.cpp +++ b/src/rtc/native_buffer.cpp @@ -17,8 +17,7 @@ int ArgbDataSize(int height, int width) { rtc::scoped_refptr NativeBuffer::Create(webrtc::VideoType video_type, int width, int height) { - return rtc::scoped_refptr( - new rtc::RefCountedObject(video_type, width, height)); + return rtc::make_ref_counted(video_type, width, height); } webrtc::VideoFrameBuffer::Type NativeBuffer::type() const { diff --git a/src/rtc/rtc_connection.cpp b/src/rtc/rtc_connection.cpp index 9cb89236..e9e7736a 100644 --- a/src/rtc/rtc_connection.cpp +++ b/src/rtc/rtc_connection.cpp @@ -41,9 +41,8 @@ class CreateSessionDescriptionThunk static rtc::scoped_refptr Create( OnSuccessFunc on_success, OnFailureFunc on_failure) { - return rtc::scoped_refptr( - new rtc::RefCountedObject( - std::move(on_success), std::move(on_failure))); + return rtc::make_ref_counted( + std::move(on_success), std::move(on_failure)); } protected: @@ -82,9 +81,8 @@ class SetSessionDescriptionThunk static rtc::scoped_refptr Create( OnSuccessFunc on_success, OnFailureFunc on_failure) { - return rtc::scoped_refptr( - new rtc::RefCountedObject( - std::move(on_success), std::move(on_failure))); + return rtc::make_ref_counted( + std::move(on_success), std::move(on_failure)); } protected: From 3fde2279f6426354c26cf345c61ae2610ad5f75c Mon Sep 17 00:00:00 2001 From: tnoho Date: Tue, 15 Mar 2022 01:00:04 +0900 Subject: [PATCH 07/70] =?UTF-8?q?C99=20=E3=82=92=E6=8C=87=E5=AE=9A?= =?UTF-8?q?=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/macos_arm64/install_deps.sh | 2 +- build/macos_x86_64/install_deps.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/macos_arm64/install_deps.sh b/build/macos_arm64/install_deps.sh index ec86b678..bf687eda 100755 --- a/build/macos_arm64/install_deps.sh +++ b/build/macos_arm64/install_deps.sh @@ -121,7 +121,7 @@ if [ $SDL2_CHANGED -eq 1 -o ! -e $INSTALL_DIR/SDL2/lib/libSDL2.a ]; then # SDL2 の CMakeLists.txt は Metal をサポートしてくれてないので、configure でビルドする # ref: https://bugzilla.libsdl.org/show_bug.cgi?id=4617 SYSROOT="`xcrun --sdk macosx --show-sdk-path`" - CC="$INSTALL_DIR/llvm/clang/bin/clang -target $ARCH_NAME -mmacosx-version-min=11.0 --sysroot=$SYSROOT" \ + CC="$INSTALL_DIR/llvm/clang/bin/clang -std=c99 -target $ARCH_NAME -mmacosx-version-min=11.0 --sysroot=$SYSROOT" \ CXX="$INSTALL_DIR/llvm/clang/bin/clang++ -target $ARCH_NAME -mmacosx-version-min=11.0 --sysroot=$SYSROOT -nostdinc++" \ $SOURCE_DIR/SDL2/source/configure --host=$ARCH_NAME --disable-shared --prefix=$INSTALL_DIR/SDL2 make -j$JOBS diff --git a/build/macos_x86_64/install_deps.sh b/build/macos_x86_64/install_deps.sh index d0450326..c494c310 100755 --- a/build/macos_x86_64/install_deps.sh +++ b/build/macos_x86_64/install_deps.sh @@ -121,7 +121,7 @@ if [ $SDL2_CHANGED -eq 1 -o ! -e $INSTALL_DIR/SDL2/lib/libSDL2.a ]; then # SDL2 の CMakeLists.txt は Metal をサポートしてくれてないので、configure でビルドする # ref: https://bugzilla.libsdl.org/show_bug.cgi?id=4617 SYSROOT="`xcrun --sdk macosx --show-sdk-path`" - CC="$INSTALL_DIR/llvm/clang/bin/clang --sysroot=$SYSROOT" \ + CC="$INSTALL_DIR/llvm/clang/bin/clang -std=c99 --sysroot=$SYSROOT" \ CXX="$INSTALL_DIR/llvm/clang/bin/clang++ --sysroot=$SYSROOT -nostdinc++" \ $SOURCE_DIR/SDL2/source/configure --disable-shared --prefix=$INSTALL_DIR/SDL2 make -j$JOBS From 312af6549c880f4083b744204855d87679d78367 Mon Sep 17 00:00:00 2001 From: tnoho Date: Tue, 15 Mar 2022 01:36:01 +0900 Subject: [PATCH 08/70] =?UTF-8?q?Revert=20"C99=20=E3=82=92=E6=8C=87?= =?UTF-8?q?=E5=AE=9A=E3=81=99=E3=82=8B"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 3fde2279f6426354c26cf345c61ae2610ad5f75c. --- build/macos_arm64/install_deps.sh | 2 +- build/macos_x86_64/install_deps.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/macos_arm64/install_deps.sh b/build/macos_arm64/install_deps.sh index bf687eda..ec86b678 100755 --- a/build/macos_arm64/install_deps.sh +++ b/build/macos_arm64/install_deps.sh @@ -121,7 +121,7 @@ if [ $SDL2_CHANGED -eq 1 -o ! -e $INSTALL_DIR/SDL2/lib/libSDL2.a ]; then # SDL2 の CMakeLists.txt は Metal をサポートしてくれてないので、configure でビルドする # ref: https://bugzilla.libsdl.org/show_bug.cgi?id=4617 SYSROOT="`xcrun --sdk macosx --show-sdk-path`" - CC="$INSTALL_DIR/llvm/clang/bin/clang -std=c99 -target $ARCH_NAME -mmacosx-version-min=11.0 --sysroot=$SYSROOT" \ + CC="$INSTALL_DIR/llvm/clang/bin/clang -target $ARCH_NAME -mmacosx-version-min=11.0 --sysroot=$SYSROOT" \ CXX="$INSTALL_DIR/llvm/clang/bin/clang++ -target $ARCH_NAME -mmacosx-version-min=11.0 --sysroot=$SYSROOT -nostdinc++" \ $SOURCE_DIR/SDL2/source/configure --host=$ARCH_NAME --disable-shared --prefix=$INSTALL_DIR/SDL2 make -j$JOBS diff --git a/build/macos_x86_64/install_deps.sh b/build/macos_x86_64/install_deps.sh index c494c310..d0450326 100755 --- a/build/macos_x86_64/install_deps.sh +++ b/build/macos_x86_64/install_deps.sh @@ -121,7 +121,7 @@ if [ $SDL2_CHANGED -eq 1 -o ! -e $INSTALL_DIR/SDL2/lib/libSDL2.a ]; then # SDL2 の CMakeLists.txt は Metal をサポートしてくれてないので、configure でビルドする # ref: https://bugzilla.libsdl.org/show_bug.cgi?id=4617 SYSROOT="`xcrun --sdk macosx --show-sdk-path`" - CC="$INSTALL_DIR/llvm/clang/bin/clang -std=c99 --sysroot=$SYSROOT" \ + CC="$INSTALL_DIR/llvm/clang/bin/clang --sysroot=$SYSROOT" \ CXX="$INSTALL_DIR/llvm/clang/bin/clang++ --sysroot=$SYSROOT -nostdinc++" \ $SOURCE_DIR/SDL2/source/configure --disable-shared --prefix=$INSTALL_DIR/SDL2 make -j$JOBS From b21ffe3298fe182051a694fb421387fd42dd870b Mon Sep 17 00:00:00 2001 From: torikizi Date: Tue, 15 Mar 2022 22:09:10 +0900 Subject: [PATCH 09/70] =?UTF-8?q?FAQ.md=20=E3=81=AB=20`turn=3Dfalse`=20?= =?UTF-8?q?=E3=81=AF=E5=AF=BE=E5=BF=9C=E3=81=97=E3=81=A6=E3=81=84=E3=81=AA?= =?UTF-8?q?=E3=81=84=E6=97=A8=E3=82=92=E8=BF=BD=E8=A8=98=E3=81=97=E3=81=BE?= =?UTF-8?q?=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/FAQ.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/FAQ.md b/doc/FAQ.md index 2c8b5590..2f9e5d26 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -120,3 +120,7 @@ Mac (arm64) から FHD でスクリーンキャプチャを配信したい場合 NVIDIA のビデオカードの NVENC を利用することで H.264 が利用可能になります。 ご利用の環境で H.264 が利用できるかどうかは `./momo --video-codec-engines` を使用して H264 の項目をご確認ください。 + +## TURN 機能を無効にできますか？ + +Momo は `turn = false` に対応していません。 \ No newline at end of file From 87c449663112a9cf113fc7fa0c7ffa21dfead866 Mon Sep 17 00:00:00 2001 From: torikizi Date: Wed, 16 Mar 2022 22:45:52 +0900 Subject: [PATCH 10/70] =?UTF-8?q?FAQ.md=20=E3=81=AB=E8=BF=BD=E8=A8=98?= =?UTF-8?q?=E3=81=97=E3=81=9F=E5=86=85=E5=AE=B9=E3=82=92=E8=A6=8B=E7=9B=B4?= =?UTF-8?q?=E3=81=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/FAQ.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/FAQ.md b/doc/FAQ.md index 2f9e5d26..65227c48 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -121,6 +121,6 @@ Mac (arm64) から FHD でスクリーンキャプチャを配信したい場合 NVIDIA のビデオカードの NVENC を利用することで H.264 が利用可能になります。 ご利用の環境で H.264 が利用できるかどうかは `./momo --video-codec-engines` を使用して H264 の項目をご確認ください。 -## TURN 機能を無効にできますか？ +## Sora モードを利用するときに Sora の TURN 機能を無効にして利用することはできますか？ -Momo は `turn = false` に対応していません。 \ No newline at end of file +Momo は Sora の TURN 機能を無効にして利用することはできません。 \ No newline at end of file From a4b249e2dcc3e43b63659c41858953060251e389 Mon Sep 17 00:00:00 2001 From: tnoho Date: Thu, 17 Mar 2022 23:02:51 +0900 Subject: [PATCH 11/70] =?UTF-8?q?mac=20=E3=81=AF=20SDL=20=E3=81=AB?= =?UTF-8?q?=E3=83=91=E3=83=83=E3=83=81=E3=82=92=E5=BD=93=E3=81=A6=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/macos_arm64/install_deps.sh | 1 + build/macos_x86_64/install_deps.sh | 1 + patch/mac_sdl.patch | 15 +++++++++++++++ 3 files changed, 17 insertions(+) create mode 100644 patch/mac_sdl.patch diff --git a/build/macos_arm64/install_deps.sh b/build/macos_arm64/install_deps.sh index ec86b678..5cef54f4 100755 --- a/build/macos_arm64/install_deps.sh +++ b/build/macos_arm64/install_deps.sh @@ -117,6 +117,7 @@ if [ $SDL2_CHANGED -eq 1 -o ! -e $INSTALL_DIR/SDL2/lib/libSDL2.a ]; then mkdir -p $SOURCE_DIR/SDL2 mkdir -p $BUILD_DIR/SDL2 ../../script/setup_sdl2.sh $SDL2_VERSION $SOURCE_DIR/SDL2 + patch -uR $SOURCE_DIR/SDL2/source/configure < ../../patch/mac_sdl.patch pushd $BUILD_DIR/SDL2 # SDL2 の CMakeLists.txt は Metal をサポートしてくれてないので、configure でビルドする # ref: https://bugzilla.libsdl.org/show_bug.cgi?id=4617 diff --git a/build/macos_x86_64/install_deps.sh b/build/macos_x86_64/install_deps.sh index d0450326..385ca87d 100755 --- a/build/macos_x86_64/install_deps.sh +++ b/build/macos_x86_64/install_deps.sh @@ -117,6 +117,7 @@ if [ $SDL2_CHANGED -eq 1 -o ! -e $INSTALL_DIR/SDL2/lib/libSDL2.a ]; then mkdir -p $SOURCE_DIR/SDL2 mkdir -p $BUILD_DIR/SDL2 ../../script/setup_sdl2.sh $SDL2_VERSION $SOURCE_DIR/SDL2 + patch -uR $SOURCE_DIR/SDL2/source/configure < ../../patch/mac_sdl.patch pushd $BUILD_DIR/SDL2 # SDL2 の CMakeLists.txt は Metal をサポートしてくれてないので、configure でビルドする # ref: https://bugzilla.libsdl.org/show_bug.cgi?id=4617 diff --git a/patch/mac_sdl.patch b/patch/mac_sdl.patch new file mode 100644 index 00000000..4ff95562 --- /dev/null +++ b/patch/mac_sdl.patch @@ -0,0 +1,15 @@ +--- configure 2022-03-17 22:44:00.181213036 +0900 ++++ configure 2022-03-17 22:42:21.470152564 +0900 +@@ -20459,9 +20459,9 @@ + $as_echo "$have_gcc_declaration_after_statement" >&6; } + CFLAGS="$save_CFLAGS" + +- #if test x$have_gcc_declaration_after_statement = xyes; then +- # EXTRA_CFLAGS="$EXTRA_CFLAGS -Wdeclaration-after-statement -Werror=declaration-after-statement" +- #fi ++ if test x$have_gcc_declaration_after_statement = xyes; then ++ EXTRA_CFLAGS="$EXTRA_CFLAGS -Wdeclaration-after-statement -Werror=declaration-after-statement" ++ fi + } + + CheckWarnAll() From 569eb4e1998508385217d32911aeb485e6bb871d Mon Sep 17 00:00:00 2001 From: tnoho Date: Thu, 17 Mar 2022 23:33:19 +0900 Subject: [PATCH 12/70] =?UTF-8?q?mac=20=E3=81=AE=E3=82=AD=E3=83=A3?= =?UTF-8?q?=E3=83=97=E3=83=81=E3=83=A3=E3=83=BC=E3=81=AB=E3=82=82=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=E3=82=92=E9=81=A9=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/mac_helper/mac_capturer.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mac_helper/mac_capturer.mm b/src/mac_helper/mac_capturer.mm index b2ec0ae5..24496f0e 100644 --- a/src/mac_helper/mac_capturer.mm +++ b/src/mac_helper/mac_capturer.mm @@ -30,7 +30,7 @@ - (void)capturer:(RTCVideoCapturer*)capturer didCaptureVideoFrame:(RTCVideoFrame*)frame { const int64_t timestamp_us = frame.timeStampNs / rtc::kNumNanosecsPerMicrosec; rtc::scoped_refptr buffer = - new rtc::RefCountedObject(frame.buffer); + rtc::make_ref_counted(frame.buffer); _capturer->OnFrame(webrtc::VideoFrame::Builder() .set_video_frame_buffer(buffer) .set_rotation(webrtc::kVideoRotation_0) From db751265e3fd7b0f9974c239717ffb4a88b3ebda Mon Sep 17 00:00:00 2001 From: tnoho Date: Thu, 17 Mar 2022 23:44:33 +0900 Subject: [PATCH 13/70] =?UTF-8?q?CHANGES=20=E3=82=92=E8=BF=BD=E8=A8=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 9260a3930bad743d1290fe367755645937caa131. --- CHANGES.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 60be6043..b7923247 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,13 @@ ## develop +- [UPDATE] `libwebrtc` を `M89.4389@{#5}` に上げる + - @tnoho +- [FIX] libwebrtc m100 で make_ref_counted を使って scoped_refptr を作るようになったので修正 + - @tnoho +- [FIX] SDL のビルドが mac では declaration-after-statement に触れてビルドが通らないのでパッチで回避 + - @tnoho + ## 2022.1.0 - [UPDATE] Raspberry Pi OS bullseye に対応 From 381814373c44a6e0b779ae25a05eae2044bad466 Mon Sep 17 00:00:00 2001 From: torikizi Date: Mon, 21 Mar 2022 11:53:01 +0900 Subject: [PATCH 14/70] =?UTF-8?q?USE=5FTEST.md=20=E3=81=AB=E8=A8=98?= =?UTF-8?q?=E8=BC=89=E3=81=95=E3=82=8C=E3=81=A6=E3=81=84=E3=82=8B=E3=82=B3?= =?UTF-8?q?=E3=83=9E=E3=83=B3=E3=83=89=E4=BE=8B=E3=82=92=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=20-=20momo=20=E5=90=8C=E5=A3=AB=E3=81=AE=E6=8E=A5=E7=B6=9A?= =?UTF-8?q?=E3=81=A7=E5=88=A9=E7=94=A8=E3=81=97=E3=81=A6=E3=81=84=E3=82=8B?= =?UTF-8?q?=E3=82=B3=E3=83=9E=E3=83=B3=E3=83=89=E4=BE=8B=E3=81=AE=E3=82=AA?= =?UTF-8?q?=E3=83=97=E3=82=B7=E3=83=A7=E3=83=B3=E3=81=8C=E4=B8=8D=E8=B6=B3?= =?UTF-8?q?=E3=81=97=E3=81=A6=E3=81=84=E3=81=9F=E3=81=AE=E3=81=A7=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=E3=81=97=E3=81=BE=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/USE_TEST.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/USE_TEST.md b/doc/USE_TEST.md index a5442e90..4b22d643 100644 --- a/doc/USE_TEST.md +++ b/doc/USE_TEST.md @@ -33,7 +33,7 @@ Momo 1: Momo 2: ```shell -./momo --use-sdl --show-me ayame ws://[Momo 1のIPアドレス]:8080/ws test +./momo --use-sdl --show-me ayame --signaling-url ws://[Momo 1のIPアドレス]:8080/ws --channel-id test ``` Google STUN を利用したくない場合は`--no-google-stun`をオプションを追加することで可能になります。 @@ -47,7 +47,7 @@ Momo 1: Momo 2: ```shell -./momo --no-google-stun --use-sdl --show-me ayame ws://[Momo 1のIPアドレス]:8080/ws test +./momo --no-google-stun --use-sdl --show-me ayame --signaling-url ws://[Momo 1のIPアドレス]:8080/ws --channel-id test ``` 配信がうまくいくとそれぞれのマシンにお互いの映像と音声が出力されます。 From 205d6c9564566d9b8ee9dcac7e0196b43e92a25b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Mar 2022 14:39:39 +0000 Subject: [PATCH 15/70] Bump actions/cache from 2 to 3 Bumps [actions/cache](https://github.com/actions/cache) from 2 to 3. - [Release notes](https://github.com/actions/cache/releases) - [Commits](https://github.com/actions/cache/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 6 +++--- .github/workflows/daily_build.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4ff288c7..8a57f01a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,7 +32,7 @@ jobs: id: versions - name: Cache Boost ${{ steps.versions.outputs.boost_version }} id: cache-boost - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: _cache\boost key: windows-boost-${{ steps.versions.outputs.boost_version }}.zip.v5 @@ -80,7 +80,7 @@ jobs: id: versions - name: Cache Boost ${{ steps.versions.outputs.boost_version }} id: cache-boost - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: _cache/boost key: macos-boost-${{ steps.versions.outputs.boost_version }}.tar.gz.v2 @@ -129,7 +129,7 @@ jobs: id: versions - name: Cache Boost ${{ steps.versions.outputs.boost_version }} id: cache-boost - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: _cache/boost key: ${{ matrix.name }}-boost-${{ steps.versions.outputs.boost_version }}.tar.gz.v2 diff --git a/.github/workflows/daily_build.yml b/.github/workflows/daily_build.yml index 1586be1d..a371e126 100644 --- a/.github/workflows/daily_build.yml +++ b/.github/workflows/daily_build.yml @@ -28,7 +28,7 @@ jobs: id: versions - name: Cache Boost ${{ steps.versions.outputs.boost_version }} id: cache-boost - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: _cache\boost key: windows-boost-${{ steps.versions.outputs.boost_version }}.zip.v5 @@ -53,7 +53,7 @@ jobs: id: versions - name: Cache Boost ${{ steps.versions.outputs.boost_version }} id: cache-boost - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: _cache/boost key: macos-boost-${{ steps.versions.outputs.boost_version }}.tar.gz.v2 @@ -83,7 +83,7 @@ jobs: id: versions - name: Cache Boost ${{ steps.versions.outputs.boost_version }} id: cache-boost - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: _cache/boost key: ${{ matrix.name }}-boost-${{ steps.versions.outputs.boost_version }}.tar.gz.v2 From c23b0d990c6f8512def2d30dfbf6874d456f1999 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Tue, 29 Mar 2022 16:30:48 +0900 Subject: [PATCH 16/70] =?UTF-8?q?CLI11=20=E3=82=92=202.2.0=20=E3=81=AB?= =?UTF-8?q?=E4=B8=8A=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 3 +++ VERSION | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 60be6043..36bb11a3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,9 @@ ## develop +- [UPDATE] CLI11 を 2.2.0 に上げる + - @voluntas + ## 2022.1.0 - [UPDATE] Raspberry Pi OS bullseye に対応 diff --git a/VERSION b/VERSION index d5d1ad99..2ea3eb07 100644 --- a/VERSION +++ b/VERSION @@ -1,7 +1,7 @@ MOMO_VERSION=2022.1.0 WEBRTC_BUILD_VERSION=99.4844.1.0 BOOST_VERSION=1.78.0 -CLI11_VERSION=2.1.2 +CLI11_VERSION=2.2.0 SDL2_VERSION=2.0.20 CMAKE_VERSION=3.22.3 CUDA_VERSION=11.0.2-1 From 105b310faa2764ff3464a8c1b9aabcad66dfb691 Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 29 Mar 2022 17:54:31 +0900 Subject: [PATCH 17/70] =?UTF-8?q?NvCodec=20(CUDA)=20=E3=81=AE=20MJPEG=20?= =?UTF-8?q?=E3=83=87=E3=82=B3=E3=83=BC=E3=83=89=E3=82=92=E3=82=A8=E3=83=B3?= =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=89=E6=99=82=E3=81=A7=E3=81=AF=E3=81=AA?= =?UTF-8?q?=E3=81=8F=E3=82=AD=E3=83=A3=E3=83=97=E3=83=81=E3=83=A3=E6=99=82?= =?UTF-8?q?=E3=81=AB=E5=A4=89=E6=9B=B4=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit サイマルキャスト時に３回 MJPEG デコードが必要になってしまうし、 そもそも既存の実装だとセグフォしてしまうため。 --- .gitignore | 2 + CMakeLists.txt | 33 ++--- src/cuda/cuda_context.h | 16 +++ src/cuda/cuda_context_cuda.cpp | 40 ++++++ src/cuda/cuda_context_cuda.h | 10 ++ src/hwenc_jetson/jetson_buffer.cpp | 75 +++++------- src/hwenc_jetson/jetson_buffer.h | 57 +++++---- src/hwenc_jetson/jetson_jpeg_decoder.cpp | 13 +- src/hwenc_jetson/jetson_jpeg_decoder.h | 11 +- src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp | 2 +- src/hwenc_jetson/jetson_jpeg_decoder_pool.h | 4 +- src/hwenc_jetson/jetson_video_decoder.cpp | 15 ++- src/hwenc_jetson/jetson_video_encoder.cpp | 105 ++++++++-------- src/hwenc_nvcodec/nvcodec_h264_encoder.cpp | 59 +++++++-- src/hwenc_nvcodec/nvcodec_h264_encoder.h | 7 ++ .../nvcodec_h264_encoder_cuda.cpp | 70 ++--------- src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.h | 9 +- .../nvcodec_mjpeg_decoder_cuda.cpp | 38 ++++++ .../nvcodec_mjpeg_decoder_cuda.h | 20 +++ src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp | 114 ++++++++++++++++++ src/hwenc_nvcodec/nvcodec_v4l2_capturer.h | 33 +++++ src/main.cpp | 23 ++++ src/p2p/p2p_websocket_session.cpp | 1 - src/rtc/momo_video_encoder_factory.cpp | 42 +++++-- src/rtc/momo_video_encoder_factory.h | 14 ++- src/rtc/rtc_connection.cpp | 3 +- src/rtc/rtc_manager.cpp | 7 +- src/rtc/rtc_manager.h | 7 ++ src/sora/sora_data_channel_on_asio.h | 7 +- src/util.cpp | 6 +- .../v4l2_video_capturer.cpp | 4 +- 31 files changed, 593 insertions(+), 254 deletions(-) create mode 100644 src/cuda/cuda_context.h create mode 100644 src/cuda/cuda_context_cuda.cpp create mode 100644 src/cuda/cuda_context_cuda.h create mode 100644 src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.cpp create mode 100644 src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.h create mode 100644 src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp create mode 100644 src/hwenc_nvcodec/nvcodec_v4l2_capturer.h diff --git a/.gitignore b/.gitignore index 172d049f..ca932ada 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,8 @@ webrtc_logs_0 *.swp /_build +/_source +/_install /_package /build/windows_*/_source /build/windows_*/_build diff --git a/CMakeLists.txt b/CMakeLists.txt index 9579de2a..22bbfc14 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -432,10 +432,10 @@ if (USE_SCREEN_CAPTURER) if (TARGET_OS STREQUAL "linux") target_link_libraries(momo PRIVATE - gio-2.0 - gobject-2.0 - glib-2.0 - Xext + # gio-2.0 + # gobject-2.0 + # glib-2.0 + # Xext Xdamage Xfixes Xcomposite @@ -568,19 +568,19 @@ elseif (TARGET_OS STREQUAL "linux") target_link_libraries(momo PRIVATE X11 - Xau - Xdmcp + # Xau + # Xdmcp Xtst - xcb - plds4 + # xcb + # plds4 Xext - expat + # expat dl - nss3 - nssutil3 - plc4 - nspr4 - rt + # nss3 + # nssutil3 + # plc4 + # nspr4 + # rt Threads::Threads ) @@ -588,8 +588,11 @@ elseif (TARGET_OS STREQUAL "linux") if (USE_NVCODEC_ENCODER) target_sources(momo PRIVATE + src/cuda/cuda_context_cuda.cpp src/hwenc_nvcodec/nvcodec_h264_encoder.cpp + src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp + src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.cpp NvCodec/NvCodec/NvDecoder/NvDecoder.cpp NvCodec/NvCodec/NvEncoder/NvEncoder.cpp NvCodec/NvCodec/NvEncoder/NvEncoderCuda.cpp) @@ -601,7 +604,9 @@ elseif (TARGET_OS STREQUAL "linux") # これらのソースは CUDA としてコンパイルする set_source_files_properties( + src/cuda/cuda_context_cuda.cpp src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp + src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.cpp NvCodec/NvCodec/NvDecoder/NvDecoder.cpp NvCodec/NvCodec/NvEncoder/NvEncoderCuda.cpp PROPERTIES diff --git a/src/cuda/cuda_context.h b/src/cuda/cuda_context.h new file mode 100644 index 00000000..4c11a9f5 --- /dev/null +++ b/src/cuda/cuda_context.h @@ -0,0 +1,16 @@ +#ifndef CUDA_CONTEXT_H_ +#define CUDA_CONTEXT_H_ + +#include + +// に依存せずに CUDA のコンテキストを表す +class CudaContext { + public: + static std::shared_ptr Create(); + void* Context() const; + + private: + std::shared_ptr impl_; +}; + +#endif \ No newline at end of file diff --git a/src/cuda/cuda_context_cuda.cpp b/src/cuda/cuda_context_cuda.cpp new file mode 100644 index 00000000..c36dca55 --- /dev/null +++ b/src/cuda/cuda_context_cuda.cpp @@ -0,0 +1,40 @@ +#include "cuda_context_cuda.h" + +#include + +#include "dyn/cuda.h" + +CUcontext GetCudaContext(std::shared_ptr ctx) { + return static_cast(ctx->Context()); +} + +struct CudaContextImpl { + CUdevice device; + CUcontext context; + ~CudaContextImpl() { dyn::cuCtxDestroy(context); } +}; + +std::shared_ptr CudaContext::Create() { + CUdevice device; + CUcontext context; + + ck(dyn::cuInit(0)); + ck(dyn::cuDeviceGet(&device, 0)); + char device_name[80]; + ck(dyn::cuDeviceGetName(device_name, sizeof(device_name), device)); + std::cout << "GPU in use: " << device_name << std::endl; + ck(dyn::cuCtxCreate(&context, 0, device)); + + std::shared_ptr impl(new CudaContextImpl()); + impl->device = device; + impl->context = context; + + std::shared_ptr ctx(new CudaContext()); + ctx->impl_ = impl; + + return ctx; +} + +void* CudaContext::Context() const { + return std::static_pointer_cast(impl_)->context; +} \ No newline at end of file diff --git a/src/cuda/cuda_context_cuda.h b/src/cuda/cuda_context_cuda.h new file mode 100644 index 00000000..b63aee38 --- /dev/null +++ b/src/cuda/cuda_context_cuda.h @@ -0,0 +1,10 @@ +#ifndef CUDA_CONTEXT_CUDA_H_ +#define CUDA_CONTEXT_CUDA_H_ + +#include + +#include "cuda_context.h" + +CUcontext GetCudaContext(std::shared_ptr ctx); + +#endif \ No newline at end of file diff --git a/src/hwenc_jetson/jetson_buffer.cpp b/src/hwenc_jetson/jetson_buffer.cpp index b2a2f759..40387a48 100644 --- a/src/hwenc_jetson/jetson_buffer.cpp +++ b/src/hwenc_jetson/jetson_buffer.cpp @@ -20,14 +20,8 @@ rtc::scoped_refptr JetsonBuffer::Create( uint32_t pixfmt, std::shared_ptr decoder) { return new rtc::RefCountedObject( - video_type, - raw_width, - raw_height, - scaled_width, - scaled_height, - fd, - pixfmt, - decoder); + video_type, raw_width, raw_height, scaled_width, scaled_height, fd, + pixfmt, decoder); } rtc::scoped_refptr JetsonBuffer::Create( @@ -37,18 +31,13 @@ rtc::scoped_refptr JetsonBuffer::Create( int scaled_width, int scaled_height) { return new rtc::RefCountedObject( - video_type, - raw_width, - raw_height, - scaled_width, - scaled_height); + video_type, raw_width, raw_height, scaled_width, scaled_height); } webrtc::VideoFrameBuffer::Type JetsonBuffer::type() const { return Type::kNative; } - webrtc::VideoType JetsonBuffer::VideoType() const { return video_type_; } @@ -118,12 +107,12 @@ rtc::scoped_refptr JetsonBuffer::ToI420() { } int ret; - void *data_addr; + void* data_addr; uint8_t* dest_addr; for (int plane = 0; plane < dmabuf_params.num_planes; plane++) { - ret = NvBufferMemMap (dmabuf_fd, plane, NvBufferMem_Read, &data_addr); + ret = NvBufferMemMap(dmabuf_fd, plane, NvBufferMem_Read, &data_addr); if (ret == 0) { - NvBufferMemSyncForCpu (dmabuf_fd, plane, &data_addr); + NvBufferMemSyncForCpu(dmabuf_fd, plane, &data_addr); int height, width; if (plane == 0) { dest_addr = scaled_buffer.get()->MutableDataY(); @@ -138,16 +127,15 @@ rtc::scoped_refptr JetsonBuffer::ToI420() { width = (scaled_width_ + 1) >> 1; height = (scaled_height_ + 1) >> 1; } - for (int i = 0; i < height; i++) - { - memcpy(dest_addr + width * i, - (uint8_t*)data_addr + dmabuf_params.pitch[plane] * i, - width); + for (int i = 0; i < height; i++) { + memcpy(dest_addr + width * i, + (uint8_t*)data_addr + dmabuf_params.pitch[plane] * i, width); } } - NvBufferMemUnMap (dmabuf_fd, plane, &data_addr); + NvBufferMemUnMap(dmabuf_fd, plane, &data_addr); if (ret == -1) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to NvBufferMemMap plane=" << plane; + RTC_LOG(LS_ERROR) << __FUNCTION__ + << " Failed to NvBufferMemMap plane=" << plane; return scaled_buffer; } } @@ -206,15 +194,14 @@ size_t JetsonBuffer::Length() const { return length_; } -JetsonBuffer::JetsonBuffer( - webrtc::VideoType video_type, - int raw_width, - int raw_height, - int scaled_width, - int scaled_height, - int fd, - uint32_t pixfmt, - std::shared_ptr decoder) +JetsonBuffer::JetsonBuffer(webrtc::VideoType video_type, + int raw_width, + int raw_height, + int scaled_width, + int scaled_height, + int fd, + uint32_t pixfmt, + std::shared_ptr decoder) : video_type_(video_type), raw_width_(raw_width), raw_height_(raw_height), @@ -223,15 +210,13 @@ JetsonBuffer::JetsonBuffer( fd_(fd), pixfmt_(pixfmt), decoder_(decoder), - data_(nullptr) { -} + data_(nullptr) {} -JetsonBuffer::JetsonBuffer( - webrtc::VideoType video_type, - int raw_width, - int raw_height, - int scaled_width, - int scaled_height) +JetsonBuffer::JetsonBuffer(webrtc::VideoType video_type, + int raw_width, + int raw_height, + int scaled_width, + int scaled_height) : video_type_(video_type), raw_width_(raw_width), raw_height_(raw_height), @@ -240,8 +225,6 @@ JetsonBuffer::JetsonBuffer( fd_(-1), pixfmt_(0), decoder_(nullptr), - data_(static_cast( - webrtc::AlignedMalloc( - webrtc::CalcBufferSize(video_type, raw_width, raw_height), - kBufferAlignment))) { -} + data_(static_cast(webrtc::AlignedMalloc( + webrtc::CalcBufferSize(video_type, raw_width, raw_height), + kBufferAlignment))) {} diff --git a/src/hwenc_jetson/jetson_buffer.h b/src/hwenc_jetson/jetson_buffer.h index e6dce37d..4290eab3 100644 --- a/src/hwenc_jetson/jetson_buffer.h +++ b/src/hwenc_jetson/jetson_buffer.h @@ -1,8 +1,8 @@ #ifndef JETSON_BUFFER_H_ #define JETSON_BUFFER_H_ -#include #include +#include // WebRTC #include @@ -18,21 +18,20 @@ class JetsonBuffer : public webrtc::VideoFrameBuffer { public: static rtc::scoped_refptr Create( - webrtc::VideoType video_type, - int raw_width, - int raw_height, - int scaled_width, - int scaled_height, - int fd, - uint32_t pixfmt, - std::shared_ptr decoder); + webrtc::VideoType video_type, + int raw_width, + int raw_height, + int scaled_width, + int scaled_height, + int fd, + uint32_t pixfmt, + std::shared_ptr decoder); - static rtc::scoped_refptr Create( - webrtc::VideoType video_type, - int raw_width, - int raw_height, - int scaled_width, - int scaled_height); + static rtc::scoped_refptr Create(webrtc::VideoType video_type, + int raw_width, + int raw_height, + int scaled_width, + int scaled_height); Type type() const override; webrtc::VideoType VideoType() const; @@ -50,22 +49,20 @@ class JetsonBuffer : public webrtc::VideoFrameBuffer { size_t Length() const; protected: - JetsonBuffer( - webrtc::VideoType video_type, - int raw_width, - int raw_height, - int scaled_width, - int scaled_height, - int fd, - uint32_t pixfmt, - std::shared_ptr decoder); + JetsonBuffer(webrtc::VideoType video_type, + int raw_width, + int raw_height, + int scaled_width, + int scaled_height, + int fd, + uint32_t pixfmt, + std::shared_ptr decoder); - JetsonBuffer( - webrtc::VideoType video_type, - int raw_width, - int raw_height, - int scaled_width, - int scaled_height); + JetsonBuffer(webrtc::VideoType video_type, + int raw_width, + int raw_height, + int scaled_width, + int scaled_height); private: webrtc::VideoType video_type_; diff --git a/src/hwenc_jetson/jetson_jpeg_decoder.cpp b/src/hwenc_jetson/jetson_jpeg_decoder.cpp index 977480cd..ea25f698 100644 --- a/src/hwenc_jetson/jetson_jpeg_decoder.cpp +++ b/src/hwenc_jetson/jetson_jpeg_decoder.cpp @@ -3,18 +3,17 @@ JetsonJpegDecoder::JetsonJpegDecoder( std::shared_ptr pool, std::unique_ptr decoder) - : pool_(pool), decoder_(std::move(decoder)) { -} + : pool_(pool), decoder_(std::move(decoder)) {} JetsonJpegDecoder::~JetsonJpegDecoder() { pool_->Push(std::move(decoder_)); } -int JetsonJpegDecoder::DecodeToFd(int &fd, - unsigned char *in_buf, +int JetsonJpegDecoder::DecodeToFd(int& fd, + unsigned char* in_buf, unsigned long in_buf_size, - uint32_t &pixfmt, - uint32_t &width, - uint32_t &height) { + uint32_t& pixfmt, + uint32_t& width, + uint32_t& height) { return decoder_->decodeToFd(fd, in_buf, in_buf_size, pixfmt, width, height); } \ No newline at end of file diff --git a/src/hwenc_jetson/jetson_jpeg_decoder.h b/src/hwenc_jetson/jetson_jpeg_decoder.h index e17399ab..a9e4af65 100644 --- a/src/hwenc_jetson/jetson_jpeg_decoder.h +++ b/src/hwenc_jetson/jetson_jpeg_decoder.h @@ -16,12 +16,15 @@ class JetsonJpegDecoder { std::unique_ptr decoder); ~JetsonJpegDecoder(); - int DecodeToFd(int &fd, - unsigned char *in_buf, unsigned long in_buf_size, - uint32_t &pixfmt, uint32_t &width, uint32_t &height); + int DecodeToFd(int& fd, + unsigned char* in_buf, + unsigned long in_buf_size, + uint32_t& pixfmt, + uint32_t& width, + uint32_t& height); + private: std::shared_ptr pool_; std::unique_ptr decoder_; - }; #endif // JETSON_JPEG_DECODER_H_ \ No newline at end of file diff --git a/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp b/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp index c3c71b89..451fe5bc 100644 --- a/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp +++ b/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp @@ -15,7 +15,7 @@ std::shared_ptr JetsonJpegDecoderPool::Pop() { decoder_queue_.pop(); } } - + std::shared_ptr decoder( new JetsonJpegDecoder(shared_from_this(), std::move(nv_decoder))); return decoder; diff --git a/src/hwenc_jetson/jetson_jpeg_decoder_pool.h b/src/hwenc_jetson/jetson_jpeg_decoder_pool.h index 2873b6f2..699502bf 100644 --- a/src/hwenc_jetson/jetson_jpeg_decoder_pool.h +++ b/src/hwenc_jetson/jetson_jpeg_decoder_pool.h @@ -12,8 +12,8 @@ class JetsonJpegDecoder; -class JetsonJpegDecoderPool : - public std::enable_shared_from_this { +class JetsonJpegDecoderPool + : public std::enable_shared_from_this { public: std::shared_ptr Pop(); void Push(std::unique_ptr decoder); diff --git a/src/hwenc_jetson/jetson_video_decoder.cpp b/src/hwenc_jetson/jetson_video_decoder.cpp index aa899c8d..fbac1e76 100644 --- a/src/hwenc_jetson/jetson_video_decoder.cpp +++ b/src/hwenc_jetson/jetson_video_decoder.cpp @@ -304,7 +304,8 @@ void JetsonVideoDecoder::CaptureLoop() { } rtc::scoped_refptr i420_buffer = - buffer_pool_.CreateI420Buffer(capture_crop_->c.width, capture_crop_->c.height); + buffer_pool_.CreateI420Buffer(capture_crop_->c.width, + capture_crop_->c.height); if (!i420_buffer.get()) { // Pool has too many pending frames. RTC_HISTOGRAM_BOOLEAN( @@ -370,11 +371,13 @@ int JetsonVideoDecoder::SetCapture() { ret = decoder_->capture_plane.getCrop(*capture_crop_.get()); INIT_ERROR(ret < 0, "Failed to getCrop at capture_plane"); - RTC_LOG(LS_INFO) << __FUNCTION__ << " plane format " << format.fmt.pix_mp.pixelformat - << " " << format.fmt.pix_mp.width << "x" << format.fmt.pix_mp.height; - RTC_LOG(LS_INFO) << __FUNCTION__ << " crop " - << capture_crop_->c.top << "x" << capture_crop_->c.left - << " " << capture_crop_->c.width << "x" << format.fmt.pix_mp.height; + RTC_LOG(LS_INFO) << __FUNCTION__ << " plane format " + << format.fmt.pix_mp.pixelformat << " " + << format.fmt.pix_mp.width << "x" + << format.fmt.pix_mp.height; + RTC_LOG(LS_INFO) << __FUNCTION__ << " crop " << capture_crop_->c.top << "x" + << capture_crop_->c.left << " " << capture_crop_->c.width + << "x" << format.fmt.pix_mp.height; if (dst_dma_fd_ != -1) { NvBufferDestroy(dst_dma_fd_); diff --git a/src/hwenc_jetson/jetson_video_encoder.cpp b/src/hwenc_jetson/jetson_video_encoder.cpp index ed438fc3..ca76d316 100644 --- a/src/hwenc_jetson/jetson_video_encoder.cpp +++ b/src/hwenc_jetson/jetson_video_encoder.cpp @@ -163,10 +163,9 @@ int32_t JetsonVideoEncoder::Release() { int32_t JetsonVideoEncoder::JetsonConfigure() { int ret = 0; - bool use_converter = use_native_ && - (width_ != raw_width_ || - height_ != raw_height_ || - decode_pixfmt_ != V4L2_PIX_FMT_YUV420M); + bool use_converter = + use_native_ && (width_ != raw_width_ || height_ != raw_height_ || + decode_pixfmt_ != V4L2_PIX_FMT_YUV420M); if (use_converter) { enc0_buffer_queue_ = new std::queue; @@ -178,8 +177,9 @@ int32_t JetsonVideoEncoder::JetsonConfigure() { decode_pixfmt_, raw_width_, raw_height_, V4L2_NV_BUFFER_LAYOUT_PITCH); INIT_ERROR(ret < 0, "Failed to converter setOutputPlaneFormat"); - ret = converter_->setCapturePlaneFormat( - V4L2_PIX_FMT_YUV420M, width_, height_, V4L2_NV_BUFFER_LAYOUT_BLOCKLINEAR); + ret = + converter_->setCapturePlaneFormat(V4L2_PIX_FMT_YUV420M, width_, height_, + V4L2_NV_BUFFER_LAYOUT_BLOCKLINEAR); INIT_ERROR(ret < 0, "Failed to converter setCapturePlaneFormat"); ret = converter_->setCropRect(0, 0, raw_width_, raw_height_); @@ -194,7 +194,7 @@ int32_t JetsonVideoEncoder::JetsonConfigure() { false); INIT_ERROR(ret < 0, "Failed to setupPlane at converter output_plane"); } - + NvBufferCreateParams create_params = {0}; create_params.width = width_; create_params.height = height_; @@ -202,13 +202,13 @@ int32_t JetsonVideoEncoder::JetsonConfigure() { create_params.payloadType = NvBufferPayload_SurfArray; create_params.colorFormat = NvBufferColorFormat_YUV420; create_params.nvbuf_tag = NvBufferTag_VIDEO_ENC; - for (int i = 0; i < CONVERTER_CAPTURE_NUM; i++) - { + for (int i = 0; i < CONVERTER_CAPTURE_NUM; i++) { ret = NvBufferCreateEx(&dmabuff_fd_[i], &create_params); INIT_ERROR(ret < 0, "Failed to NvBufferCreateEx at converter"); } - - ret = converter_->capture_plane.reqbufs(V4L2_MEMORY_DMABUF, CONVERTER_CAPTURE_NUM); + + ret = converter_->capture_plane.reqbufs(V4L2_MEMORY_DMABUF, + CONVERTER_CAPTURE_NUM); INIT_ERROR(ret < 0, "Failed to reqbufs at converter capture_plane"); ret = converter_->output_plane.setStreamStatus(true); @@ -262,7 +262,7 @@ int32_t JetsonVideoEncoder::JetsonConfigure() { INIT_ERROR(ret < 0, "Failed to setHWPresetType"); } else if (codec_.codecType == webrtc::kVideoCodecVP8) { uint32_t qp_min = - codec_.mode == webrtc::VideoCodecMode::kScreensharing ? 12 : 2; + codec_.mode == webrtc::VideoCodecMode::kScreensharing ? 12 : 2; uint32_t qp_max = 56; if (codec_.qpMax >= qp_min) { qp_max = codec_.qpMax; @@ -304,12 +304,12 @@ int32_t JetsonVideoEncoder::JetsonConfigure() { if (use_native_) { if (use_dmabuff_ || use_converter) { - ret = - encoder_->output_plane.setupPlane(V4L2_MEMORY_DMABUF, 1, false, false); + ret = encoder_->output_plane.setupPlane(V4L2_MEMORY_DMABUF, 1, false, + false); INIT_ERROR(ret < 0, "Failed to setupPlane at encoder output_plane"); } else { - ret = - encoder_->output_plane.setupPlane(V4L2_MEMORY_USERPTR, 1, false, false); + ret = encoder_->output_plane.setupPlane(V4L2_MEMORY_USERPTR, 1, false, + false); INIT_ERROR(ret < 0, "Failed to setupPlane at encoder output_plane"); } } else { @@ -395,10 +395,8 @@ void JetsonVideoEncoder::JetsonRelease() { encoder_ = nullptr; if (converter_) { converter_->capture_plane.waitForDQThread(2000); - for(int i = 0; i < CONVERTER_CAPTURE_NUM; i++) - { - if(dmabuff_fd_[i] != 0) - { + for (int i = 0; i < CONVERTER_CAPTURE_NUM; i++) { + if (dmabuff_fd_[i] != 0) { NvBufferDestroy(dmabuff_fd_[i]); } } @@ -453,10 +451,11 @@ bool JetsonVideoEncoder::ConvertFinishedCallback(struct v4l2_buffer* v4l2_buf, RTC_LOG(LS_ERROR) << __FUNCTION__ << " v4l2_buf is null"; return false; } - + { std::unique_lock lock(enc0_buffer_mtx_); - enc0_buffer_cond_.wait(lock, [this] { return !enc0_buffer_queue_->empty(); }); + enc0_buffer_cond_.wait(lock, + [this] { return !enc0_buffer_queue_->empty(); }); enc0_buffer = enc0_buffer_queue_->front(); enc0_buffer_queue_->pop(); } @@ -577,10 +576,9 @@ bool JetsonVideoEncoder::EncodeFinishedCallback(struct v4l2_buffer* v4l2_buf, v4l2_ctrl_videoenc_outputbuf_metadata enc_metadata; if (encoder_->getMetadata(v4l2_buf->index, enc_metadata) != 0) { - RTC_LOG(LS_WARNING) - << __FUNCTION__ - << "getMetadata failed. SkipFrame timestamp:" - << timestamp; + RTC_LOG(LS_WARNING) << __FUNCTION__ + << "getMetadata failed. SkipFrame timestamp:" + << timestamp; return true; } @@ -751,8 +749,7 @@ int32_t JetsonVideoEncoder::Encode( frame_buffer->width(), frame_buffer->height(), input_frame.render_time_ms(), input_frame.ntp_time_ms(), input_frame.timestamp_us(), input_frame.timestamp(), - input_frame.rotation(), input_frame.color_space(), - decoder)); + input_frame.rotation(), input_frame.color_space(), decoder)); } struct v4l2_buffer v4l2_buf; @@ -766,7 +763,8 @@ int32_t JetsonVideoEncoder::Encode( NvBuffer* buffer; if (native_input_elem_->output_plane.getNumQueuedBuffers() == native_input_elem_->output_plane.getNumBuffers()) { - if (native_input_elem_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) { + if (native_input_elem_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, + 10) < 0) { RTC_LOG(LS_ERROR) << "Failed to dqBuffer at converter output_plane"; return WEBRTC_VIDEO_CODEC_ERROR; } @@ -782,34 +780,33 @@ int32_t JetsonVideoEncoder::Encode( } else if (video_type == webrtc::VideoType::kYUY2 || video_type == webrtc::VideoType::kUYVY) { buffer->planes[0].bytesused = buffer->planes[0].fmt.width * - buffer->planes[0].fmt.bytesperpixel * - buffer->planes[0].fmt.height; + buffer->planes[0].fmt.bytesperpixel * + buffer->planes[0].fmt.height; buffer->planes[0].data = native_data; } else if (video_type == webrtc::VideoType::kI420) { size_t offset = 0; - for (int i = 0; i < buffer->n_planes; i++) - { + for (int i = 0; i < buffer->n_planes; i++) { buffer->planes[i].bytesused = buffer->planes[i].fmt.width * - buffer->planes[i].fmt.bytesperpixel * - buffer->planes[i].fmt.height; + buffer->planes[i].fmt.bytesperpixel * + buffer->planes[i].fmt.height; buffer->planes[i].data = native_data + offset; offset += buffer->planes[i].bytesused; } } else if (video_type == webrtc::VideoType::kYV12) { size_t offset = 0; buffer->planes[0].bytesused = buffer->planes[0].fmt.width * - buffer->planes[0].fmt.bytesperpixel * - buffer->planes[0].fmt.height; + buffer->planes[0].fmt.bytesperpixel * + buffer->planes[0].fmt.height; buffer->planes[0].data = native_data; offset += buffer->planes[0].bytesused; buffer->planes[2].bytesused = buffer->planes[1].fmt.width * - buffer->planes[1].fmt.bytesperpixel * - buffer->planes[1].fmt.height; + buffer->planes[1].fmt.bytesperpixel * + buffer->planes[1].fmt.height; buffer->planes[2].data = native_data + offset; offset += buffer->planes[2].bytesused; buffer->planes[1].bytesused = buffer->planes[2].fmt.width * - buffer->planes[2].fmt.bytesperpixel * - buffer->planes[2].fmt.height; + buffer->planes[2].fmt.bytesperpixel * + buffer->planes[2].fmt.height; buffer->planes[1].data = native_data + offset; } else { RTC_LOG(LS_ERROR) << "Unsupported webrtc::VideoType"; @@ -897,10 +894,11 @@ int32_t JetsonVideoEncoder::Encode( return WEBRTC_VIDEO_CODEC_OK; } -int32_t JetsonVideoEncoder::SendFrame(unsigned char* buffer, - size_t size, - std::unique_ptr params, - v4l2_ctrl_videoenc_outputbuf_metadata enc_metadata) { +int32_t JetsonVideoEncoder::SendFrame( + unsigned char* buffer, + size_t size, + std::unique_ptr params, + v4l2_ctrl_videoenc_outputbuf_metadata enc_metadata) { if (!callback_) { RTC_LOG(LS_WARNING) << "InitEncode() has been called, but a callback function " @@ -925,7 +923,7 @@ int32_t JetsonVideoEncoder::SendFrame(unsigned char* buffer, webrtc::CodecSpecificInfo codec_specific; codec_specific.codecType = codec_.codecType; if (codec_.codecType == webrtc::kVideoCodecH264) { - auto encoded_image_buffer = + auto encoded_image_buffer = webrtc::EncodedImageBuffer::Create(buffer, size); encoded_image_.SetEncodedData(encoded_image_buffer); @@ -942,7 +940,7 @@ int32_t JetsonVideoEncoder::SendFrame(unsigned char* buffer, buffer += 12; size -= 12; - auto encoded_image_buffer = + auto encoded_image_buffer = webrtc::EncodedImageBuffer::Create(buffer, size); encoded_image_.SetEncodedData(encoded_image_buffer); @@ -969,8 +967,10 @@ int32_t JetsonVideoEncoder::SendFrame(unsigned char* buffer, codec_specific.codecSpecific.VP9.end_of_picture = true; codec_specific.codecSpecific.VP9.spatial_layer_resolution_present = false; if (codec_specific.codecSpecific.VP9.ss_data_available) { - codec_specific.codecSpecific.VP9.spatial_layer_resolution_present = true; - codec_specific.codecSpecific.VP9.width[0] = encoded_image_._encodedWidth; + codec_specific.codecSpecific.VP9.spatial_layer_resolution_present = + true; + codec_specific.codecSpecific.VP9.width[0] = + encoded_image_._encodedWidth; codec_specific.codecSpecific.VP9.height[0] = encoded_image_._encodedHeight; codec_specific.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_); @@ -979,11 +979,10 @@ int32_t JetsonVideoEncoder::SendFrame(unsigned char* buffer, } RTC_LOG(LS_VERBOSE) << "key_frame=" << enc_metadata.KeyFrame - << " size=" << size - << " qp=" << encoded_image_.qp_; + << " size=" << size << " qp=" << encoded_image_.qp_; - webrtc::EncodedImageCallback::Result result = callback_->OnEncodedImage( - encoded_image_, &codec_specific); + webrtc::EncodedImageCallback::Result result = + callback_->OnEncodedImage(encoded_image_, &codec_specific); if (result.error != webrtc::EncodedImageCallback::Result::OK) { RTC_LOG(LS_ERROR) << __FUNCTION__ << " OnEncodedImage failed error:" << result.error; diff --git a/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp b/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp index 0886b9bc..7fa5da62 100644 --- a/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp +++ b/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp @@ -1,10 +1,14 @@ #include "nvcodec_h264_encoder.h" // WebRTC +#include #include #include #include +// NvCodec +#include + #include "rtc/native_buffer.h" #ifdef __linux__ @@ -24,8 +28,14 @@ struct nal_entry { using Microsoft::WRL::ComPtr; #endif -NvCodecH264Encoder::NvCodecH264Encoder(const cricket::VideoCodec& codec) - : bitrate_adjuster_(0.5, 0.95) { +#ifdef __linux__ +NvCodecH264Encoder::NvCodecH264Encoder(const cricket::VideoCodec& codec, + std::shared_ptr cc) + : cuda_context_(cc), +#else +NvCodecH264Encoder::NvCodecH264Encoder(const cricket::VideoCodec& codec), +#endif + bitrate_adjuster_(0.5, 0.95) { #ifdef _WIN32 ComPtr idxgi_factory; RTC_CHECK(!FAILED(CreateDXGIFactory1(__uuidof(IDXGIFactory1), @@ -47,7 +57,7 @@ NvCodecH264Encoder::NvCodecH264Encoder(const cricket::VideoCodec& codec) RTC_LOG(INFO) << __FUNCTION__ << "GPU in use: " << szDesc; #endif #ifdef __linux__ - cuda_.reset(new NvCodecH264EncoderCuda()); + cuda_.reset(new NvCodecH264EncoderCuda(cuda_context_)); #endif } @@ -122,6 +132,22 @@ int32_t NvCodecH264Encoder::Encode( return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } +#if defined(__linux__) + if (frame.video_frame_buffer()->type() == + webrtc::VideoFrameBuffer::Type::kNV12) { + if (!use_native_) { + ReleaseNvEnc(); + use_native_ = true; + InitNvEnc(); + } + } else { + if (use_native_) { + ReleaseNvEnc(); + use_native_ = false; + InitNvEnc(); + } + } +#else if (frame.video_frame_buffer()->type() == webrtc::VideoFrameBuffer::Type::kNative) { if (!use_native_) { @@ -136,6 +162,7 @@ int32_t NvCodecH264Encoder::Encode( InitNvEnc(); } } +#endif bool send_key_frame = false; @@ -218,13 +245,27 @@ int32_t NvCodecH264Encoder::Encode( id3d11_context_->CopyResource(nv11_texture, id3d11_texture_.Get()); #endif #ifdef __linux__ + + //RTC_LOG(LS_INFO) << "type=" + // << VideoFrameBufferTypeToString( + // frame.video_frame_buffer()->type()) + // << " width_=" << width_ << " height_=" << height_ + // << " frame_width=" << frame.video_frame_buffer()->width() + // << " frame_height=" << frame.video_frame_buffer()->height(); + if (frame.video_frame_buffer()->type() == - webrtc::VideoFrameBuffer::Type::kNative) { - NativeBuffer* native_buffer = - dynamic_cast(frame.video_frame_buffer().get()); - cuda_->CopyNative(nv_encoder_.get(), native_buffer->Data(), - native_buffer->Length(), native_buffer->width(), - native_buffer->height()); + webrtc::VideoFrameBuffer::Type::kNV12) { + webrtc::NV12Buffer* buffer = + static_cast(frame.video_frame_buffer().get()); + try { + cuda_->Copy(nv_encoder_.get(), buffer->DataY(), width_, height_); + } catch (const NVDECException& e) { + RTC_LOG(LS_ERROR) << e.what(); + return WEBRTC_VIDEO_CODEC_ERROR; + } catch (const NVENCException& e) { + RTC_LOG(LS_ERROR) << e.what(); + return WEBRTC_VIDEO_CODEC_ERROR; + } } else { rtc::scoped_refptr frame_buffer = frame.video_frame_buffer()->ToI420(); diff --git a/src/hwenc_nvcodec/nvcodec_h264_encoder.h b/src/hwenc_nvcodec/nvcodec_h264_encoder.h index 41326dac..209906bf 100644 --- a/src/hwenc_nvcodec/nvcodec_h264_encoder.h +++ b/src/hwenc_nvcodec/nvcodec_h264_encoder.h @@ -22,12 +22,18 @@ #include #endif #ifdef __linux__ +#include "cuda/cuda_context.h" #include "nvcodec_h264_encoder_cuda.h" #endif class NvCodecH264Encoder : public webrtc::VideoEncoder { public: +#ifdef __linux__ + explicit NvCodecH264Encoder(const cricket::VideoCodec& codec, + std::shared_ptr cc); +#else explicit NvCodecH264Encoder(const cricket::VideoCodec& codec); +#endif ~NvCodecH264Encoder() override; static bool IsSupported(); @@ -63,6 +69,7 @@ class NvCodecH264Encoder : public webrtc::VideoEncoder { std::unique_ptr nv_encoder_; #endif #ifdef __linux__ + std::shared_ptr cuda_context_; std::unique_ptr cuda_; std::unique_ptr nv_encoder_; #endif diff --git a/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp b/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp index 3519e73e..28b26197 100644 --- a/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp +++ b/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp @@ -6,6 +6,7 @@ #include #include +#include "cuda/cuda_context_cuda.h" #include "dyn/cuda.h" // どこかにグローバルな logger の定義が必要 @@ -14,25 +15,18 @@ simplelogger::Logger* logger = class NvCodecH264EncoderCudaImpl { public: - NvCodecH264EncoderCudaImpl(); + NvCodecH264EncoderCudaImpl(std::shared_ptr ctx); ~NvCodecH264EncoderCudaImpl(); void Copy(NvEncoder* nv_encoder, const void* ptr, int width, int height); - void CopyNative(NvEncoder* nv_encoder, - const void* ptr, - int size, - int width, - int height); NvEncoder* CreateNvEncoder(int width, int height, bool use_native); private: - NvDecoder* nv_decoder_ = nullptr; - CUdevice cu_device_; - CUcontext cu_context_; + std::shared_ptr cuda_context_; }; -NvCodecH264EncoderCuda::NvCodecH264EncoderCuda() - : impl_(new NvCodecH264EncoderCudaImpl()) {} +NvCodecH264EncoderCuda::NvCodecH264EncoderCuda(std::shared_ptr ctx) + : impl_(new NvCodecH264EncoderCudaImpl(ctx)) {} NvCodecH264EncoderCuda::~NvCodecH264EncoderCuda() { delete impl_; } @@ -43,13 +37,6 @@ void NvCodecH264EncoderCuda::Copy(NvEncoder* nv_encoder, int height) { impl_->Copy(nv_encoder, ptr, width, height); } -void NvCodecH264EncoderCuda::CopyNative(NvEncoder* nv_encoder, - const void* ptr, - int size, - int width, - int height) { - impl_->CopyNative(nv_encoder, ptr, size, width, height); -} NvEncoder* NvCodecH264EncoderCuda::CreateNvEncoder(int width, int height, bool use_native) { @@ -57,7 +44,6 @@ NvEncoder* NvCodecH264EncoderCuda::CreateNvEncoder(int width, } void ShowEncoderCapability() { - ck(dyn::cuInit(0)); int nGpu = 0; ck(dyn::cuDeviceGetCount(&nGpu)); if (nGpu == 0) { @@ -164,59 +150,29 @@ void ShowEncoderCapability() { } } -NvCodecH264EncoderCudaImpl::NvCodecH264EncoderCudaImpl() { +NvCodecH264EncoderCudaImpl::NvCodecH264EncoderCudaImpl( + std::shared_ptr ctx) { ShowEncoderCapability(); - - ck(dyn::cuInit(0)); - ck(dyn::cuDeviceGet(&cu_device_, 0)); - char device_name[80]; - ck(dyn::cuDeviceGetName(device_name, sizeof(device_name), cu_device_)); - std::cout << "GPU in use: " << device_name << std::endl; - ck(dyn::cuCtxCreate(&cu_context_, 0, cu_device_)); -} -NvCodecH264EncoderCudaImpl::~NvCodecH264EncoderCudaImpl() { - if (nv_decoder_ != nullptr) { - delete nv_decoder_; - } - dyn::cuCtxDestroy(cu_context_); + cuda_context_ = ctx; } +NvCodecH264EncoderCudaImpl::~NvCodecH264EncoderCudaImpl() {} void NvCodecH264EncoderCudaImpl::Copy(NvEncoder* nv_encoder, const void* ptr, int width, int height) { const NvEncInputFrame* input_frame = nv_encoder->GetNextInputFrame(); + CUcontext context = GetCudaContext(cuda_context_); NvEncoderCuda::CopyToDeviceFrame( - cu_context_, (void*)ptr, 0, (CUdeviceptr)input_frame->inputPtr, + context, (void*)ptr, 0, (CUdeviceptr)input_frame->inputPtr, (int)input_frame->pitch, width, height, CU_MEMORYTYPE_HOST, input_frame->bufferFormat, input_frame->chromaOffsets, input_frame->numChromaPlanes); } -void NvCodecH264EncoderCudaImpl::CopyNative(NvEncoder* nv_encoder, - const void* ptr, - int size, - int width, - int height) { - if (nv_decoder_ == nullptr) { - std::cout << "Use JPEG Decoder" << std::endl; - nv_decoder_ = new NvDecoder(cu_context_, true, cudaVideoCodec_JPEG, false, - true, nullptr, nullptr, 3840, 2160); - } - int frame_count = nv_decoder_->Decode((const uint8_t*)ptr, size); - - for (int i = 0; i < frame_count; i++) { - uint8_t* frame = nv_decoder_->GetFrame(); - const NvEncInputFrame* input_frame = nv_encoder->GetNextInputFrame(); - NvEncoderCuda::CopyToDeviceFrame( - cu_context_, frame, nv_decoder_->GetDeviceFramePitch(), - (CUdeviceptr)input_frame->inputPtr, (int)input_frame->pitch, width, - height, CU_MEMORYTYPE_DEVICE, input_frame->bufferFormat, - input_frame->chromaOffsets, input_frame->numChromaPlanes); - } -} NvEncoder* NvCodecH264EncoderCudaImpl::CreateNvEncoder(int width, int height, bool use_native) { NV_ENC_BUFFER_FORMAT nvenc_format = use_native ? NV_ENC_BUFFER_FORMAT_NV12 : NV_ENC_BUFFER_FORMAT_IYUV; - return new NvEncoderCuda(cu_context_, width, height, nvenc_format); + CUcontext context = GetCudaContext(cuda_context_); + return new NvEncoderCuda(context, width, height, nvenc_format); } diff --git a/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.h b/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.h index 561d61a7..54c61aaf 100644 --- a/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.h +++ b/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.h @@ -9,19 +9,16 @@ #include +#include "cuda/cuda_context.h" + class NvCodecH264EncoderCudaImpl; class NvCodecH264EncoderCuda { public: - NvCodecH264EncoderCuda(); + NvCodecH264EncoderCuda(std::shared_ptr ctx); ~NvCodecH264EncoderCuda(); void Copy(NvEncoder* nv_encoder, const void* ptr, int width, int height); - void CopyNative(NvEncoder* nv_encoder, - const void* ptr, - int size, - int width, - int height); // 念のため も include せずポインタを利用する NvEncoder* CreateNvEncoder(int width, int height, bool use_native); diff --git a/src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.cpp b/src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.cpp new file mode 100644 index 00000000..159e3397 --- /dev/null +++ b/src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.cpp @@ -0,0 +1,38 @@ +#include "nvcodec_mjpeg_decoder_cuda.h" + +// NvCodec +#include + +#include "cuda/cuda_context_cuda.h" + +NvCodecMjpegDecoderCuda::NvCodecMjpegDecoderCuda( + std::shared_ptr ctx) + : impl_(new NvDecoder(GetCudaContext(ctx), + false, + cudaVideoCodec_JPEG, + false, + true, + nullptr, + nullptr, + 3840, + 2160)) {} + +static NvDecoder* GetDecoder(std::shared_ptr impl) { + return std::static_pointer_cast(impl).get(); +} + +int NvCodecMjpegDecoderCuda::Decode(const uint8_t* pData, int nSize) { + return GetDecoder(impl_)->Decode(pData, nSize); +} + +uint8_t* NvCodecMjpegDecoderCuda::GetFrame() { + return GetDecoder(impl_)->GetFrame(); +} + +uint8_t* NvCodecMjpegDecoderCuda::GetLockedFrame() { + return GetDecoder(impl_)->GetLockedFrame(); +} + +void NvCodecMjpegDecoderCuda::UnlockFrame(uint8_t* pFrame) { + return GetDecoder(impl_)->UnlockFrame(&pFrame); +} \ No newline at end of file diff --git a/src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.h b/src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.h new file mode 100644 index 00000000..fd154ee1 --- /dev/null +++ b/src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.h @@ -0,0 +1,20 @@ +#ifndef NVCODEC_MJPEG_DECODER_CUDA_H_ +#define NVCODEC_MJPEG_DECODER_CUDA_H_ + +#include + +#include "cuda/cuda_context.h" + +class NvCodecMjpegDecoderCuda { + public: + NvCodecMjpegDecoderCuda(std::shared_ptr ctx); + int Decode(const uint8_t* pData, int nSize); + uint8_t* GetFrame(); + uint8_t* GetLockedFrame(); + void UnlockFrame(uint8_t* pFrame); + + private: + std::shared_ptr impl_; +}; + +#endif // NVCODEC_MJPEG_DECODER_CUDA_H_ \ No newline at end of file diff --git a/src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp b/src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp new file mode 100644 index 00000000..1c48e4c3 --- /dev/null +++ b/src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp @@ -0,0 +1,114 @@ +#include "nvcodec_v4l2_capturer.h" + +// Linux +#include + +// WebRTC +#include +#include +#include + +rtc::scoped_refptr NvCodecV4L2Capturer::Create( + NvCodecV4L2CapturerConfig config) { + rtc::scoped_refptr capturer; + std::unique_ptr device_info( + webrtc::VideoCaptureFactory::CreateDeviceInfo()); + if (!device_info) { + RTC_LOG(LS_ERROR) << "Failed to CreateDeviceInfo"; + return nullptr; + } + + LogDeviceList(device_info.get()); + + for (int i = 0; i < device_info->NumberOfDevices(); ++i) { + capturer = Create(device_info.get(), config, i); + if (capturer) { + RTC_LOG(LS_INFO) << "Get Capture"; + return capturer; + } + } + RTC_LOG(LS_ERROR) << "Failed to create NvCodecV4L2Capturer"; + return nullptr; +} + +bool NvCodecV4L2Capturer::UseNativeBuffer() { + return true; +} + +rtc::scoped_refptr NvCodecV4L2Capturer::Create( + webrtc::VideoCaptureModule::DeviceInfo* device_info, + NvCodecV4L2CapturerConfig config, + size_t capture_device_index) { + char device_name[256]; + char unique_name[256]; + + if (device_info->GetDeviceName(static_cast(capture_device_index), + device_name, sizeof(device_name), unique_name, + sizeof(unique_name)) != 0) { + RTC_LOG(LS_WARNING) << "Failed to GetDeviceName"; + return nullptr; + } + + rtc::scoped_refptr v4l2_capturer( + new rtc::RefCountedObject()); + + v4l2_capturer->decoder_.reset( + new NvCodecMjpegDecoderCuda(config.cuda_context)); + + if (v4l2_capturer->Init((const char*)&unique_name, config.video_device) < 0) { + RTC_LOG(LS_WARNING) << "Failed to create NvCodecV4L2Capturer(" + << unique_name << ")"; + return nullptr; + } + + if (v4l2_capturer->StartCapture(config) < 0) { + RTC_LOG(LS_WARNING) << "Failed to start NvCodecV4L2Capturer(w = " + << config.width << ", h = " << config.height + << ", fps = " << config.framerate << ")"; + return nullptr; + } + + return v4l2_capturer; +} + +void NvCodecV4L2Capturer::OnCaptured(uint8_t* data, uint32_t bytesused) { + const int64_t timestamp_us = rtc::TimeMicros(); + int adapted_width, adapted_height, crop_width, crop_height, crop_x, crop_y; + if (!AdaptFrame(_currentWidth, _currentHeight, timestamp_us, &adapted_width, + &adapted_height, &crop_width, &crop_height, &crop_x, + &crop_y)) { + return; + } + + auto count = decoder_->Decode(data, bytesused); + for (int i = 0; i < count; i++) { + // 頑張ればここのコピーは無くせるけど、ひとまずは妥協しておく + auto buffer = webrtc::NV12Buffer::Create(_currentWidth, _currentHeight); + uint8_t* p = decoder_->GetFrame(); + //RTC_LOG(LS_INFO) << "_currentWidth=" << _currentWidth + // << " _currentHeight=" << _currentHeight + // << " height=" << buffer->height() + // << " StrideY=" << buffer->StrideY() + // << " ChromaHeight=" << buffer->ChromaHeight() + // << " StrideUV=" << buffer->StrideUV(); + std::memcpy(buffer->MutableDataY(), p, + buffer->height() * buffer->StrideY() + + buffer->ChromaHeight() * buffer->StrideUV()); + + rtc::scoped_refptr buf; + if (_currentWidth != adapted_width || _currentHeight != adapted_height) { + buf = buffer->CropAndScale(crop_x, crop_y, crop_width, crop_height, + adapted_width, adapted_height); + } else { + buf = buffer; + } + + OnFrame(webrtc::VideoFrame::Builder() + .set_video_frame_buffer(buf) + .set_timestamp_rtp(0) + .set_timestamp_ms(rtc::TimeMillis()) + .set_timestamp_us(rtc::TimeMicros()) + .set_rotation(webrtc::kVideoRotation_0) + .build()); + } +} \ No newline at end of file diff --git a/src/hwenc_nvcodec/nvcodec_v4l2_capturer.h b/src/hwenc_nvcodec/nvcodec_v4l2_capturer.h new file mode 100644 index 00000000..a518031c --- /dev/null +++ b/src/hwenc_nvcodec/nvcodec_v4l2_capturer.h @@ -0,0 +1,33 @@ +#ifndef NVCODEC_V4L2_CAPTURER_H_ +#define NVCODEC_V4L2_CAPTURER_H_ + +#include "v4l2_video_capturer/v4l2_video_capturer.h" + +#include "./nvcodec_mjpeg_decoder_cuda.h" + +struct NvCodecV4L2CapturerConfig : V4L2VideoCapturerConfig { + NvCodecV4L2CapturerConfig(const V4L2VideoCapturerConfig& config) { + *static_cast(this) = config; + } + std::shared_ptr cuda_context; +}; + +class NvCodecV4L2Capturer : public V4L2VideoCapturer { + public: + static rtc::scoped_refptr Create( + NvCodecV4L2CapturerConfig config); + + bool UseNativeBuffer() override; + + private: + static rtc::scoped_refptr Create( + webrtc::VideoCaptureModule::DeviceInfo* device_info, + NvCodecV4L2CapturerConfig config, + size_t capture_device_index); + + void OnCaptured(uint8_t* data, uint32_t bytesused) override; + + std::shared_ptr decoder_; +}; + +#endif // NVCODEC_V4L2_CAPTURER_H_ diff --git a/src/main.cpp b/src/main.cpp index 8abd81ed..c267718a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -21,6 +21,8 @@ #include "hwenc_mmal/mmal_v4l2_capturer.h" #elif USE_JETSON_ENCODER #include "hwenc_jetson/jetson_v4l2_capturer.h" +#elif USE_NVCODEC_ENCODER +#include "hwenc_nvcodec/nvcodec_v4l2_capturer.h" #endif #include "v4l2_video_capturer/v4l2_video_capturer.h" #else @@ -45,6 +47,10 @@ #include #endif +#if defined(__linux__) && USE_NVCODEC_ENCODER +#include "cuda/cuda_context.h" +#endif + const size_t kDefaultMaxLogFileSize = 10 * 1024 * 1024; int main(int argc, char* argv[]) { @@ -80,6 +86,10 @@ int main(int argc, char* argv[]) { } rtc::LogMessage::AddLogToStream(log_sink.get(), rtc::LS_INFO); +#if defined(__linux__) && USE_NVCODEC_ENCODER + auto cuda_context = CudaContext::Create(); +#endif + auto capturer = ([&]() -> rtc::scoped_refptr { if (args.no_video_device) { return nullptr; @@ -128,6 +138,14 @@ int main(int argc, char* argv[]) { } else { return V4L2VideoCapturer::Create(std::move(v4l2_config)); } +#elif USE_NVCODEC_ENCODER + if (v4l2_config.use_native) { + NvCodecV4L2CapturerConfig nvcodec_config = v4l2_config; + nvcodec_config.cuda_context = cuda_context; + return NvCodecV4L2Capturer::Create(std::move(nvcodec_config)); + } else { + return V4L2VideoCapturer::Create(std::move(v4l2_config)); + } #else return V4L2VideoCapturer::Create(std::move(v4l2_config)); #endif @@ -171,6 +189,11 @@ int main(int argc, char* argv[]) { rtcm_config.h264_decoder = args.h264_decoder; rtcm_config.priority = args.priority; + +#if defined(__linux__) && USE_NVCODEC_ENCODER + rtcm_config.cuda_context = cuda_context; +#endif + #if USE_SDL2 std::unique_ptr sdl_renderer = nullptr; if (args.use_sdl) { diff --git a/src/p2p/p2p_websocket_session.cpp b/src/p2p/p2p_websocket_session.cpp index 23df0ed0..bdc88166 100644 --- a/src/p2p/p2p_websocket_session.cpp +++ b/src/p2p/p2p_websocket_session.cpp @@ -87,7 +87,6 @@ void P2PWebsocketSession::OnRead(boost::system::error_code ec, ~Guard() { f(); } } guard = {[this]() { DoRead(); }}; - RTC_LOG(LS_INFO) << __FUNCTION__ << ": recv_string=" << recv_string; boost::json::error_code jec; diff --git a/src/rtc/momo_video_encoder_factory.cpp b/src/rtc/momo_video_encoder_factory.cpp index 462cd1e4..c246b716 100644 --- a/src/rtc/momo_video_encoder_factory.cpp +++ b/src/rtc/momo_video_encoder_factory.cpp @@ -6,9 +6,9 @@ #include #include #include +#include #include #include -#include #include #include #include @@ -39,19 +39,34 @@ MomoVideoEncoderFactory::MomoVideoEncoderFactory( VideoCodecInfo::Type av1_encoder, VideoCodecInfo::Type h264_encoder, bool simulcast, - bool hardware_encoder_only) + bool hardware_encoder_only +#if defined(__linux__) && USE_NVCODEC_ENCODER + , + std::shared_ptr cuda_context +#endif + ) : vp8_encoder_(vp8_encoder), vp9_encoder_(vp9_encoder), av1_encoder_(av1_encoder), h264_encoder_(h264_encoder), - hardware_encoder_only_(hardware_encoder_only) { + hardware_encoder_only_(hardware_encoder_only) +#if defined(__linux__) && USE_NVCODEC_ENCODER + , + cuda_context_(cuda_context) +#endif +{ #if defined(__APPLE__) video_encoder_factory_ = CreateObjCEncoderFactory(); #endif if (simulcast) { internal_encoder_factory_.reset(new MomoVideoEncoderFactory( vp8_encoder, vp9_encoder, av1_encoder, h264_encoder, false, - hardware_encoder_only)); + hardware_encoder_only +#if defined(__linux__) && USE_NVCODEC_ENCODER + , + cuda_context_ +#endif + )); } } std::vector @@ -232,10 +247,21 @@ MomoVideoEncoderFactory::CreateVideoEncoder( #if USE_NVCODEC_ENCODER if (h264_encoder_ == VideoCodecInfo::Type::NVIDIA && NvCodecH264Encoder::IsSupported()) { - return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { - return std::unique_ptr( - absl::make_unique(cricket::VideoCodec(format))); - }); + return WithSimulcast( +#if defined(__linux__) + format, + [cuda_context = cuda_context_](const webrtc::SdpVideoFormat& format) { + return std::unique_ptr( + absl::make_unique( + cricket::VideoCodec(format), cuda_context)); +#else + format, + [](const webrtc::SdpVideoFormat& format) { + return std::unique_ptr( + absl::make_unique( + cricket::VideoCodec(format))); +#endif + }); } #endif } diff --git a/src/rtc/momo_video_encoder_factory.h b/src/rtc/momo_video_encoder_factory.h index 587e3907..cd007c5a 100644 --- a/src/rtc/momo_video_encoder_factory.h +++ b/src/rtc/momo_video_encoder_factory.h @@ -11,6 +11,10 @@ #include "video_codec_info.h" +#if defined(__linux__) && USE_NVCODEC_ENCODER +#include "cuda/cuda_context.h" +#endif + class MomoVideoEncoderFactory : public webrtc::VideoEncoderFactory { VideoCodecInfo::Type vp8_encoder_; VideoCodecInfo::Type vp9_encoder_; @@ -26,7 +30,12 @@ class MomoVideoEncoderFactory : public webrtc::VideoEncoderFactory { VideoCodecInfo::Type av1_encoder, VideoCodecInfo::Type h264_encoder, bool simulcast, - bool hardware_encoder_only); + bool hardware_encoder_only +#if defined(__linux__) && USE_NVCODEC_ENCODER + , + std::shared_ptr cuda_context +#endif + ); virtual ~MomoVideoEncoderFactory() {} std::vector GetSupportedFormats() const override; @@ -39,6 +48,9 @@ class MomoVideoEncoderFactory : public webrtc::VideoEncoderFactory { const webrtc::SdpVideoFormat& format, std::function( const webrtc::SdpVideoFormat&)> create); +#if defined(__linux__) && USE_NVCODEC_ENCODER + std::shared_ptr cuda_context_; +#endif }; #endif // MOMO_VIDEO_ENCODER_FACTORY_H_ diff --git a/src/rtc/rtc_connection.cpp b/src/rtc/rtc_connection.cpp index bf4c37eb..82674f6f 100644 --- a/src/rtc/rtc_connection.cpp +++ b/src/rtc/rtc_connection.cpp @@ -123,7 +123,8 @@ void RTCConnection::CreateOffer(OnCreateSuccessFunc on_success, webrtc::DataChannelInit config; auto result = connection_->CreateDataChannelOrError("serial", &config); if (!result.ok()) { - RTC_LOG(LS_ERROR) << "CreateDataChannel() failed: " << result.error().message(); + RTC_LOG(LS_ERROR) << "CreateDataChannel() failed: " + << result.error().message(); } data_manager->OnDataChannel(result.MoveValue()); } diff --git a/src/rtc/rtc_manager.cpp b/src/rtc/rtc_manager.cpp index fe6aad54..24e165bf 100644 --- a/src/rtc/rtc_manager.cpp +++ b/src/rtc/rtc_manager.cpp @@ -103,7 +103,12 @@ RTCManager::RTCManager( resolve(cf.vp9_encoder, info.vp9_encoders), resolve(cf.av1_encoder, info.av1_encoders), resolve(cf.h264_encoder, info.h264_encoders), cf.simulcast, - cf.hardware_encoder_only)); + cf.hardware_encoder_only +#if defined(__linux__) && USE_NVCODEC_ENCODER + , + cf.cuda_context +#endif + )); media_dependencies.video_decoder_factory = std::unique_ptr( absl::make_unique( diff --git a/src/rtc/rtc_manager.h b/src/rtc/rtc_manager.h index 6ff13e9f..68bb13f5 100644 --- a/src/rtc/rtc_manager.h +++ b/src/rtc/rtc_manager.h @@ -1,10 +1,13 @@ #ifndef RTC_MANAGER_H_ #define RTC_MANAGER_H_ +#include + // WebRTC #include #include +#include "cuda/cuda_context.h" #include "rtc_connection.h" #include "rtc_data_manager_dispatcher.h" #include "rtc_message_sender.h" @@ -49,6 +52,10 @@ struct RTCManagerConfig { } return webrtc::DegradationPreference::BALANCED; } + +#if defined(__linux__) && USE_NVCODEC_ENCODER + std::shared_ptr cuda_context; +#endif }; class RTCManager { diff --git a/src/sora/sora_data_channel_on_asio.h b/src/sora/sora_data_channel_on_asio.h index 745f12e3..3baf906a 100644 --- a/src/sora/sora_data_channel_on_asio.h +++ b/src/sora/sora_data_channel_on_asio.h @@ -37,8 +37,11 @@ class SoraDataChannelOnAsio : public RTCDataManager { void Send(std::string label, const webrtc::DataBuffer& data) { dc_.Send(label, data); } - void Close(const webrtc::DataBuffer& disconnect_message, std::function on_close, int disconnect_wait_timeout = 5) { - timer_.expires_from_now(boost::posix_time::seconds(disconnect_wait_timeout)); + void Close(const webrtc::DataBuffer& disconnect_message, + std::function on_close, + int disconnect_wait_timeout = 5) { + timer_.expires_from_now( + boost::posix_time::seconds(disconnect_wait_timeout)); timer_.async_wait([on_close](const boost::system::error_code& ec) { if (ec == boost::asio::error::operation_aborted) { return; diff --git a/src/util.cpp b/src/util.cpp index 603c3388..fb8eae1d 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -406,11 +406,11 @@ void Util::ParseArgs(int argc, << std::endl; std::cout << std::endl; std::cout << "USE_MMAL_ENCODER=" BOOST_PP_STRINGIZE(USE_MMAL_ENCODER) - << std::endl; + << std::endl; std::cout << "USE_JETSON_ENCODER=" BOOST_PP_STRINGIZE(USE_JETSON_ENCODER) - << std::endl; + << std::endl; std::cout << "USE_NVCODEC_ENCODER=" BOOST_PP_STRINGIZE(USE_NVCODEC_ENCODER) - << std::endl; + << std::endl; std::cout << "USE_SDL2=" BOOST_PP_STRINGIZE(USE_SDL2) << std::endl; exit(0); } diff --git a/src/v4l2_video_capturer/v4l2_video_capturer.cpp b/src/v4l2_video_capturer/v4l2_video_capturer.cpp index 9839a0fc..596cc813 100644 --- a/src/v4l2_video_capturer/v4l2_video_capturer.cpp +++ b/src/v4l2_video_capturer/v4l2_video_capturer.cpp @@ -493,8 +493,8 @@ bool V4L2VideoCapturer::CaptureProcess() { // JPEG の先頭は SOI マーカー 0xffd8 で始まるのでチェックして落ちないようにする。 if (_captureVideoType == webrtc::VideoType::kMJPEG && bytesused >= 2) { if (data[0] != 0xff || data[1] != 0xd8) { - RTC_LOG(LS_WARNING) << __FUNCTION__ - << " Invalid JPEG buffer frame skipped"; + RTC_LOG(LS_WARNING) + << __FUNCTION__ << " Invalid JPEG buffer frame skipped"; } else { unsigned int eosSearchSize = MJPEG_EOS_SEARCH_SIZE; uint8_t* p; From 658a3ee55f7353a3c28264fb4bcd205ce057829b Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 29 Mar 2022 18:00:21 +0900 Subject: [PATCH 18/70] =?UTF-8?q?VSCode=20=E3=81=AE=E8=A8=AD=E5=AE=9A?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=92=E5=85=A5=E3=82=8C?= =?UTF-8?q?=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 - .vscode/c_cpp_properties.json | 37 +++++++++++++++++++++++++++++++++++ .vscode/settings.json | 5 +++++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 .vscode/c_cpp_properties.json create mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index ca932ada..53070f00 100644 --- a/.gitignore +++ b/.gitignore @@ -33,7 +33,6 @@ momo core -.vscode *.tar.gz webrtc_logs_0 .DS_Store diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json new file mode 100644 index 00000000..7f43dc48 --- /dev/null +++ b/.vscode/c_cpp_properties.json @@ -0,0 +1,37 @@ +{ + "configurations": [ + { + "name": "Ubuntu 20.04 x86_64", + "includePath": [ + "${workspaceFolder}/src", + "${workspaceFolder}/NvCodec/include", + "${workspaceFolder}/NvCodec/NvCodec", + "${workspaceFolder}/_install/ubuntu-20.04_x86_64/webrtc/include", + "${workspaceFolder}/_install/ubuntu-20.04_x86_64/webrtc/include/third_party/abseil-cpp", + "${workspaceFolder}/_install/ubuntu-20.04_x86_64/webrtc/include/third_party/boringssl/src/include", + "${workspaceFolder}/_install/ubuntu-20.04_x86_64/webrtc/include/third_party/libyuv/include", + "${workspaceFolder}/_install/ubuntu-20.04_x86_64/webrtc/include/third_party/zlib", + "${workspaceFolder}/_install/ubuntu-20.04_x86_64/boost/include", + "${workspaceFolder}/_install/ubuntu-20.04_x86_64/CLI11/include", + "${workspaceFolder}/_install/ubuntu-20.04_x86_64/SDL2/include/SDL2", + "${workspaceFolder}/_build/ubuntu-20.04_x86_64", + "/usr/local/cuda/include" + ], + "defines": [ + "WEBRTC_LINUX", + "USE_NVCODEC_ENCODER=1", + "USE_MMAL_ENCODER=0", + "USE_JETSON_ENCODER=0", + "USE_H264=1", + "USE_SDL2=1", + "USE_LINUX_PULSE_AUDIO=0", + "USE_SCREEN_CAPTURER=1" + ], + "compilerPath": "/usr/bin/clang", + "cStandard": "c11", + "cppStandard": "c++14", + "intelliSenseMode": "linux-clang-x64" + } + ], + "version": 4 +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..6bf2d0e8 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "[cpp]": { + "editor.formatOnSave": true + } +} \ No newline at end of file From 404bfe1fda2ae2c47c1c524fd22f621203659776 Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 29 Mar 2022 18:44:00 +0900 Subject: [PATCH 19/70] =?UTF-8?q?Windows=20=E3=81=A7=E3=81=AE=E3=82=B3?= =?UTF-8?q?=E3=83=B3=E3=83=91=E3=82=A4=E3=83=AB=E3=82=A8=E3=83=A9=E3=83=BC?= =?UTF-8?q?=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/c_cpp_properties.json | 30 ++++++++++++++++++++++ src/hwenc_nvcodec/nvcodec_h264_encoder.cpp | 6 ++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 7f43dc48..3f83baec 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -1,5 +1,35 @@ { "configurations": [ + { + "name": "Windows", + "includePath": [ + "${workspaceFolder}/src", + "${workspaceFolder}/NvCodec/include", + "${workspaceFolder}/NvCodec/NvCodec", + "${workspaceFolder}/build/windows_x86_64/_install/webrtc/include", + "${workspaceFolder}/build/windows_x86_64/_install/webrtc/include/third_party/abseil-cpp", + "${workspaceFolder}/build/windows_x86_64/_install/webrtc/include/third_party/boringssl/src/include", + "${workspaceFolder}/build/windows_x86_64/_install/webrtc/include/third_party/libyuv/include", + "${workspaceFolder}/build/windows_x86_64/_install/webrtc/include/third_party/zlib", + "${workspaceFolder}/build/windows_x86_64/_install/boost/include", + "${workspaceFolder}/build/windows_x86_64/_install/CLI11/include", + "${workspaceFolder}/build/windows_x86_64/_install/SDL2/include/SDL2", + "${workspaceFolder}/_build/windows_x86_64" + ], + "defines": [ + "WEBRTC_WINDOWS", + "USE_NVCODEC_ENCODER=1", + "USE_MMAL_ENCODER=0", + "USE_JETSON_ENCODER=0", + "USE_H264=1", + "USE_SDL2=1", + "USE_LINUX_PULSE_AUDIO=0", + "USE_SCREEN_CAPTURER=1" + ], + "cStandard": "c11", + "cppStandard": "c++14", + "intelliSenseMode": "windows-msvc-x64" + }, { "name": "Ubuntu 20.04 x86_64", "includePath": [ diff --git a/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp b/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp index 7fa5da62..cd255001 100644 --- a/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp +++ b/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp @@ -6,9 +6,6 @@ #include #include -// NvCodec -#include - #include "rtc/native_buffer.h" #ifdef __linux__ @@ -33,7 +30,8 @@ NvCodecH264Encoder::NvCodecH264Encoder(const cricket::VideoCodec& codec, std::shared_ptr cc) : cuda_context_(cc), #else -NvCodecH264Encoder::NvCodecH264Encoder(const cricket::VideoCodec& codec), +NvCodecH264Encoder::NvCodecH264Encoder(const cricket::VideoCodec& codec) + : #endif bitrate_adjuster_(0.5, 0.95) { #ifdef _WIN32 From db246c1c17e80a4091cfb73a32a4008e1e2f8a44 Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 29 Mar 2022 19:02:51 +0900 Subject: [PATCH 20/70] =?UTF-8?q?NVDECException=20=E3=81=AF=E7=99=BA?= =?UTF-8?q?=E7=94=9F=E3=81=97=E3=81=AA=E3=81=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_nvcodec/nvcodec_h264_encoder.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp b/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp index cd255001..7741d8e1 100644 --- a/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp +++ b/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp @@ -257,9 +257,6 @@ int32_t NvCodecH264Encoder::Encode( static_cast(frame.video_frame_buffer().get()); try { cuda_->Copy(nv_encoder_.get(), buffer->DataY(), width_, height_); - } catch (const NVDECException& e) { - RTC_LOG(LS_ERROR) << e.what(); - return WEBRTC_VIDEO_CODEC_ERROR; } catch (const NVENCException& e) { RTC_LOG(LS_ERROR) << e.what(); return WEBRTC_VIDEO_CODEC_ERROR; From 11bd3199856e57776fea7f2da4ce728319f65a23 Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 29 Mar 2022 19:05:16 +0900 Subject: [PATCH 21/70] =?UTF-8?q?CHANGES=20=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 60be6043..8909d1a2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,9 @@ ## develop +- [FIX] Ubuntu 20.04 + H.264 + サイマルキャスト + --hw-mjpeg-decoder true で落ちるのを修正 (#221) + - @melpon + ## 2022.1.0 - [UPDATE] Raspberry Pi OS bullseye に対応 From 908051e93b1f85fb1a6ba6d96ccedce8802871ba Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 30 Mar 2022 18:09:45 +0900 Subject: [PATCH 22/70] =?UTF-8?q?=E3=83=AD=E3=83=BC=E3=82=AB=E3=83=AB?= =?UTF-8?q?=E3=83=93=E3=83=AB=E3=83=89=E7=94=A8=E3=82=B9=E3=82=AF=E3=83=AA?= =?UTF-8?q?=E3=83=97=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- localbuild/build.ubuntu-20.04_x86_64.sh | 168 ++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100755 localbuild/build.ubuntu-20.04_x86_64.sh diff --git a/localbuild/build.ubuntu-20.04_x86_64.sh b/localbuild/build.ubuntu-20.04_x86_64.sh new file mode 100755 index 00000000..45c189ff --- /dev/null +++ b/localbuild/build.ubuntu-20.04_x86_64.sh @@ -0,0 +1,168 @@ +#!/bin/bash + +cd `dirname $0` + +set -ex + +# apt install cuda=$CUDA_VERSION clang-10 curl git libxtst-dev libxdamage-dev libxfixes-dev libxrandr-dev libxcomposite-dev + +PACKAGE_NAME=ubuntu-20.04_x86_64 +PROJECT_DIR=`pwd`/.. +SCRIPT_DIR=$PROJECT_DIR/script +SOURCE_DIR=$PROJECT_DIR/_source/$PACKAGE_NAME +BUILD_DIR=$PROJECT_DIR/_build/$PACKAGE_NAME +INSTALL_DIR=$PROJECT_DIR/_install/$PACKAGE_NAME + +mkdir -p $SOURCE_DIR +mkdir -p $BUILD_DIR +mkdir -p $INSTALL_DIR + +source $PROJECT_DIR/VERSION +CURRENT_VERSION="`cat $PROJECT_DIR/VERSION`" +INSTALLED_VERSION="" +if [ -e $INSTALL_DIR/VERSION ]; then + INSTALLED_VERSION="`cat $INSTALL_DIR/VERSION`" +fi + +if [ "$CURRENT_VERSION" != "$INSTALLED_VERSION" ]; then + # WebRTC の取得 + $SCRIPT_DIR/get_webrtc.sh "$WEBRTC_BUILD_VERSION" ubuntu-20.04_x86_64 $INSTALL_DIR $SOURCE_DIR + + # コンパイラの取得 + $SCRIPT_DIR/get_llvm.sh $INSTALL_DIR/webrtc $INSTALL_DIR + + # Boost のビルド + $SCRIPT_DIR/setup_boost.sh $BOOST_VERSION $SOURCE_DIR/boost $SOURCE_DIR + pushd $SOURCE_DIR/boost/source + mkdir -p $BUILD_DIR/boost + echo "using clang : : $INSTALL_DIR/llvm/clang/bin/clang++ : ;" > project-config.jam + ./b2 \ + cxxstd=17 \ + cxxflags=" \ + -D_LIBCPP_ABI_UNSTABLE \ + -D_LIBCPP_DISABLE_AVAILABILITY \ + -nostdinc++ \ + -isystem${INSTALL_DIR}/llvm/libcxx/include \ + " \ + linkflags=' \ + ' \ + toolset=clang \ + visibility=global \ + target-os=linux \ + address-model=64 \ + link=static \ + variant=release \ + install \ + -j`nproc` \ + --build-dir=$BUILD_DIR/boost \ + --ignore-site-config \ + --prefix=$INSTALL_DIR/boost \ + --with-filesystem \ + --with-json + + # CLI11 の取得 + rm -rf $INSTALL_DIR/CLI11 + git clone --branch v$CLI11_VERSION --depth 1 https://github.com/CLIUtils/CLI11.git $INSTALL_DIR/CLI11 + + # CMake のインストール + $SCRIPT_DIR/get_cmake.sh "$CMAKE_VERSION" linux $INSTALL_DIR + export PATH="$INSTALL_DIR/cmake/bin:$PATH" + + # SDL2 のビルド + $SCRIPT_DIR/setup_sdl2.sh "$SDL2_VERSION" $SOURCE_DIR/sdl2 + mkdir -p $BUILD_DIR/sdl2/ + pushd $BUILD_DIR/sdl2 + cmake $SOURCE_DIR/sdl2/source \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR/SDL2 \ + -DCMAKE_C_COMPILER=$INSTALL_DIR/llvm/clang/bin/clang \ + -DCMAKE_CXX_COMPILER=$INSTALL_DIR/llvm/clang/bin/clang++ \ + -DBUILD_SHARED_LIBS=OFF \ + -DSDL_ATOMIC=OFF \ + -DSDL_AUDIO=OFF \ + -DSDL_VIDEO=ON \ + -DSDL_RENDER=ON \ + -DSDL_EVENTS=ON \ + -DSDL_JOYSTICK=ON \ + -DSDL_HAPTIC=ON \ + -DSDL_POWER=ON \ + -DSDL_THREADS=ON \ + -DSDL_TIMERS=OFF \ + -DSDL_FILE=OFF \ + -DSDL_LOADSO=ON \ + -DSDL_CPUINFO=OFF \ + -DSDL_FILESYSTEM=OFF \ + -DSDL_DLOPEN=ON \ + -DSDL_SENSOR=ON \ + -DVIDEO_OPENGL=ON \ + -DVIDEO_OPENGLES=ON \ + -DVIDEO_RPI=OFF \ + -DVIDEO_WAYLAND=OFF \ + -DVIDEO_X11=ON \ + -DX11_SHARED=OFF \ + -DVIDEO_X11_XCURSOR=OFF \ + -DVIDEO_X11_XINERAMA=OFF \ + -DVIDEO_X11_XINPUT=OFF \ + -DVIDEO_X11_XRANDR=OFF \ + -DVIDEO_X11_XSCRNSAVER=OFF \ + -DVIDEO_X11_XSHAPE=OFF \ + -DVIDEO_X11_XVM=OFF \ + -DVIDEO_VULKAN=OFF \ + -DVIDEO_VIVANTE=OFF \ + -DVIDEO_COCOA=OFF \ + -DVIDEO_METAL=OFF \ + -DVIDEO_KMSDRM=OFF + make -j`nproc` + make install + popd + +# CUDA 周りのインストール +# apt-get update +# apt-get install -y software-properties-common +# wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin +# mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 +# apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub +# add-apt-repository "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" +# apt-get update +# DEBIAN_FRONTEND=noninteractive apt-get -y install cuda=$CUDA_VERSION clang-10 + + cp $PROJECT_DIR/VERSION $INSTALL_DIR/VERSION +fi + +source $INSTALL_DIR/webrtc/VERSIONS + +export PATH="$INSTALL_DIR/cmake/bin:$PATH" + +pushd $PROJECT_DIR + MOMO_COMMIT="`git rev-parse HEAD`" + MOMO_COMMIT_SHORT="`cat $MOMO_COMMIT | cut -b 1-8`" +popd + +mkdir -p $BUILD_DIR/momo +pushd $BUILD_DIR/momo + cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DMOMO_VERSION=$MOMO_VERSION \ + -DMOMO_COMMIT=$MOMO_COMMIT \ + -DWEBRTC_BUILD_VERSION=$WEBRTC_BUILD_VERSION \ + -DWEBRTC_READABLE_VERSION="$WEBRTC_READABLE_VERSION" \ + -DWEBRTC_COMMIT="$WEBRTC_COMMIT" \ + -DBOOST_ROOT_DIR=$INSTALL_DIR/boost \ + -DCLI11_ROOT_DIR=$INSTALL_DIR/CLI11 \ + -DSDL2_ROOT_DIR=$INSTALL_DIR/SDL2 \ + -DWEBRTC_INCLUDE_DIR=$INSTALL_DIR/webrtc/include \ + -DWEBRTC_LIBRARY_DIR=$INSTALL_DIR/webrtc/lib \ + -DLIBCXX_INCLUDE_DIR=$INSTALL_DIR/llvm/libcxx/include \ + -DTARGET_OS="linux" \ + -DTARGET_OS_LINUX="ubuntu-20.04" \ + -DTARGET_ARCH="x86_64" \ + -DUSE_H264=ON \ + -DUSE_SDL2=ON \ + -DUSE_NVCODEC_ENCODER=ON \ + -DUSE_SCREEN_CAPTURER=ON \ + -DCMAKE_C_COMPILER=clang-10 \ + -DCMAKE_CXX_COMPILER=clang++-10 \ + -DUSE_LIBCXX=ON \ + $PROJECT_DIR + cmake --build . -j`nproc` +popd From f8b4dc74206694f1dddb4a22555c55a07dea5bea Mon Sep 17 00:00:00 2001 From: melpon Date: Thu, 31 Mar 2022 19:03:59 +0900 Subject: [PATCH 23/70] WIP --- .vscode/c_cpp_properties.json | 32 ++++ localbuild/build.raspberry-pi-os_armv7.sh | 182 ++++++++++++++++++++++ src/hwenc_mmal/mmal_v4l2_capturer.cpp | 19 ++- 3 files changed, 232 insertions(+), 1 deletion(-) create mode 100755 localbuild/build.raspberry-pi-os_armv7.sh diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 3f83baec..ee5311fa 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -61,6 +61,38 @@ "cStandard": "c11", "cppStandard": "c++14", "intelliSenseMode": "linux-clang-x64" + }, + { + "name": "raspberry-pi-os_armv7 on Ubuntu 20.04 x86_64", + "includePath": [ + "${workspaceFolder}/src", + "${workspaceFolder}/NvCodec/include", + "${workspaceFolder}/NvCodec/NvCodec", + "${workspaceFolder}/_install/raspberry-pi-os_armv7/webrtc/include", + "${workspaceFolder}/_install/raspberry-pi-os_armv7/webrtc/include/third_party/abseil-cpp", + "${workspaceFolder}/_install/raspberry-pi-os_armv7/webrtc/include/third_party/boringssl/src/include", + "${workspaceFolder}/_install/raspberry-pi-os_armv7/webrtc/include/third_party/libyuv/include", + "${workspaceFolder}/_install/raspberry-pi-os_armv7/webrtc/include/third_party/zlib", + "${workspaceFolder}/_install/raspberry-pi-os_armv7/boost/include", + "${workspaceFolder}/_install/raspberry-pi-os_armv7/CLI11/include", + "${workspaceFolder}/_install/raspberry-pi-os_armv7/SDL2/include/SDL2", + "${workspaceFolder}/_install/raspberry-pi-os_armv7/rootfs/usr/include", + "${workspaceFolder}/_build/raspberry-pi-os_armv7", + ], + "defines": [ + "WEBRTC_LINUX", + "USE_NVCODEC_ENCODER=0", + "USE_MMAL_ENCODER=1", + "USE_JETSON_ENCODER=0", + "USE_H264=1", + "USE_SDL2=1", + "USE_LINUX_PULSE_AUDIO=0", + "USE_SCREEN_CAPTURER=0" + ], + "compilerPath": "${workspaceFolder}/_install/raspberry-pi-os_armv7/llvm/clang/bin/clang", + "cStandard": "c11", + "cppStandard": "c++14", + "intelliSenseMode": "linux-clang-x64" } ], "version": 4 diff --git a/localbuild/build.raspberry-pi-os_armv7.sh b/localbuild/build.raspberry-pi-os_armv7.sh new file mode 100755 index 00000000..f3050d61 --- /dev/null +++ b/localbuild/build.raspberry-pi-os_armv7.sh @@ -0,0 +1,182 @@ +#!/bin/bash + +# Ubuntu 20.04 x86_64 のローカルでビルドするスクリプト + +cd `dirname $0` + +set -ex + +# apt install curl git + +PACKAGE_NAME=raspberry-pi-os_armv7 +PROJECT_DIR=`pwd`/.. +SCRIPT_DIR=$PROJECT_DIR/script +BUILDBASE_DIR=$PROJECT_DIR/build/$PACKAGE_NAME +SOURCE_DIR=$PROJECT_DIR/_source/$PACKAGE_NAME +BUILD_DIR=$PROJECT_DIR/_build/$PACKAGE_NAME +INSTALL_DIR=$PROJECT_DIR/_install/$PACKAGE_NAME + +mkdir -p $SOURCE_DIR +mkdir -p $BUILD_DIR +mkdir -p $INSTALL_DIR + +source $PROJECT_DIR/VERSION +CURRENT_VERSION="`cat $PROJECT_DIR/VERSION`" +INSTALLED_VERSION="" +if [ -e $INSTALL_DIR/VERSION ]; then + INSTALLED_VERSION="`cat $INSTALL_DIR/VERSION`" +fi + +if [ "$CURRENT_VERSION" != "$INSTALLED_VERSION" ]; then + # RootFS の構築 + $SCRIPT_DIR/init_rootfs_armhf.sh $INSTALL_DIR/rootfs $BUILDBASE_DIR/rpi-raspbian.conf + + # WebRTC の取得 + $SCRIPT_DIR/get_webrtc.sh "$WEBRTC_BUILD_VERSION" raspberry-pi-os_armv7 $INSTALL_DIR $SOURCE_DIR + + # コンパイラの取得 + $SCRIPT_DIR/get_llvm.sh $INSTALL_DIR/webrtc $INSTALL_DIR + + # Boost のビルド + $SCRIPT_DIR/setup_boost.sh $BOOST_VERSION $SOURCE_DIR/boost $SOURCE_DIR + pushd $SOURCE_DIR/boost/source + mkdir -p $BUILD_DIR/boost + echo "using clang : : $INSTALL_DIR/llvm/clang/bin/clang++ : ;" > project-config.jam + ./b2 \ + cxxstd=17 \ + cxxflags=" \ + -D_LIBCPP_ABI_UNSTABLE \ + -D_LIBCPP_DISABLE_AVAILABILITY \ + -nostdinc++ \ + -isystem${INSTALL_DIR}/llvm/libcxx/include \ + --target=arm-linux-gnueabihf \ + --sysroot=$INSTALL_DIR/rootfs \ + -I$INSTALL_DIR/rootfs/usr/include/arm-linux-gnueabihf \ + " \ + linkflags=' \ + ' \ + toolset=clang \ + visibility=global \ + target-os=linux \ + architecture=arm \ + address-model=32 \ + link=static \ + variant=release \ + install \ + -j`nproc` \ + --build-dir=$BUILD_DIR/boost \ + --ignore-site-config \ + --prefix=$INSTALL_DIR/boost \ + --with-filesystem \ + --with-json + + # CLI11 の取得 + rm -rf $INSTALL_DIR/CLI11 + git clone --branch v$CLI11_VERSION --depth 1 https://github.com/CLIUtils/CLI11.git $INSTALL_DIR/CLI11 + + # CMake のインストール + $SCRIPT_DIR/get_cmake.sh "$CMAKE_VERSION" linux $INSTALL_DIR + export PATH="$INSTALL_DIR/cmake/bin:$PATH" + + # SDL2 のビルド + $SCRIPT_DIR/setup_sdl2.sh "$SDL2_VERSION" $SOURCE_DIR/sdl2 + mkdir -p $BUILD_DIR/sdl2/ + pushd $BUILD_DIR/sdl2 + cmake $SOURCE_DIR/sdl2/source \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR/SDL2 \ + -DCMAKE_C_COMPILER=$INSTALL_DIR/llvm/clang/bin/clang \ + -DCMAKE_C_COMPILER_TARGET=arm-linux-gnueabihf \ + -DCMAKE_CXX_COMPILER=$INSTALL_DIR/llvm/clang/bin/clang++ \ + -DCMAKE_CXX_COMPILER_TARGET=arm-linux-gnueabihf \ + -DCMAKE_SYSTEM_NAME=Linux \ + -DCMAKE_SYSTEM_PROCESSOR=arm \ + -DCMAKE_FIND_ROOT_PATH=$INSTALL_DIR/rootfs \ + -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \ + -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=BOTH \ + -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH \ + -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH \ + -DCMAKE_SYSROOT=$INSTALL_DIR/rootfs \ + -DSDL_STATIC=ON \ + -DSDL_SHARED=OFF \ + -DSDL_ATOMIC=OFF \ + -DSDL_AUDIO=OFF \ + -DSDL_VIDEO=ON \ + -DSDL_RENDER=ON \ + -DSDL_EVENTS=ON \ + -DSDL_JOYSTICK=ON \ + -DSDL_HAPTIC=ON \ + -DSDL_POWER=ON \ + -DSDL_THREADS=ON \ + -DSDL_TIMERS=OFF \ + -DSDL_FILE=OFF \ + -DSDL_LOADSO=ON \ + -DSDL_CPUINFO=OFF \ + -DSDL_FILESYSTEM=OFF \ + -DSDL_DLOPEN=ON \ + -DSDL_SENSOR=ON \ + -DSDL_COCOA=OFF \ + -DSDL_KMSDRM=OFF \ + -DSDL_METAL=OFF \ + -DSDL_OPENGL=ON \ + -DSDL_OPENGLES=ON \ + -DSDL_RPI=OFF \ + -DSDL_VIVANTE=OFF \ + -DSDL_VULKAN=OFF \ + -DSDL_WAYLAND=OFF \ + -DSDL_X11=ON \ + -DSDL_X11_SHARED=OFF \ + -DSDL_X11_XCURSOR=OFF \ + -DSDL_X11_XFIXES=OFF \ + -DSDL_X11_XINERAMA=OFF \ + -DSDL_X11_XINPUT=OFF \ + -DSDL_X11_XRANDR=OFF \ + -DSDL_X11_XSCRNSAVER=OFF \ + -DSDL_X11_XSHAPE=OFF \ + -DSDL_X11_XVM=OFF + make -j`nproc` + make install + popd + + cp $PROJECT_DIR/VERSION $INSTALL_DIR/VERSION +fi + +source $INSTALL_DIR/webrtc/VERSIONS + +export PATH="$INSTALL_DIR/cmake/bin:$PATH" + +pushd $PROJECT_DIR + MOMO_COMMIT="`git rev-parse HEAD`" + MOMO_COMMIT_SHORT="`cat $MOMO_COMMIT | cut -b 1-8`" +popd + +mkdir -p $BUILD_DIR/momo +pushd $BUILD_DIR/momo + cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DMOMO_VERSION=$MOMO_VERSION \ + -DMOMO_COMMIT=$MOMO_COMMIT \ + -DWEBRTC_BUILD_VERSION=$WEBRTC_BUILD_VERSION \ + -DWEBRTC_READABLE_VERSION="$WEBRTC_READABLE_VERSION" \ + -DWEBRTC_COMMIT="$WEBRTC_COMMIT" \ + -DBOOST_ROOT_DIR=$INSTALL_DIR/boost \ + -DCLI11_ROOT_DIR=$INSTALL_DIR/CLI11 \ + -DSDL2_ROOT_DIR=$INSTALL_DIR/SDL2 \ + -DWEBRTC_INCLUDE_DIR=$INSTALL_DIR/webrtc/include \ + -DWEBRTC_LIBRARY_DIR=$INSTALL_DIR/webrtc/lib \ + -DLIBCXX_INCLUDE_DIR=$INSTALL_DIR/llvm/libcxx/include \ + -DTARGET_OS="linux" \ + -DTARGET_OS_LINUX="raspberry-pi-os" \ + -DTARGET_ARCH="arm" \ + -DTARGET_ARCH_ARM="armv7" \ + -DUSE_MMAL_ENCODER=ON \ + -DUSE_H264=ON \ + -DUSE_SDL2=ON \ + -DCLANG_ROOT=$INSTALL_DIR/llvm/clang \ + -DUSE_LIBCXX=ON \ + -DSYSROOT=$INSTALL_DIR/rootfs \ + -DLIBCXX_INCLUDE_DIR=$INSTALL_DIR/llvm/libcxx/include \ + -DBoost_ARCHITECTURE=32 \ + $PROJECT_DIR + cmake --build . -j`nproc` +popd diff --git a/src/hwenc_mmal/mmal_v4l2_capturer.cpp b/src/hwenc_mmal/mmal_v4l2_capturer.cpp index e552ad46..840da7b1 100644 --- a/src/hwenc_mmal/mmal_v4l2_capturer.cpp +++ b/src/hwenc_mmal/mmal_v4l2_capturer.cpp @@ -113,11 +113,26 @@ void MMALV4L2Capturer::OnCaptured(uint8_t* data, uint32_t bytesused) { } std::lock_guard lock(mtx_); + //{ + // static int n = 0; + // char file[255]; + // sprintf(file, "mjpeg/mjpeg%04d.jpg", n); + // n += 1; + // FILE* fp = fopen(file, "wb"); + // if (fp == NULL) { + // RTC_LOG(LS_ERROR) << "Failed to open " << file; + // return; + // } + // fwrite(data, 1, bytesused, fp); + // fclose(fp); + //} + if (configured_width_ != adapted_width || configured_height_ != adapted_height) { RTC_LOG(LS_INFO) << "Resizer reinitialized from " << configured_width_ << "x" << configured_height_ << " to " << adapted_width - << "x" << adapted_height; + << "x" << adapted_height << " (current " << _currentWidth + << "x" << _currentHeight << ")"; MMALRelease(); if (MMALConfigure(adapted_width, adapted_height) == -1) { RTC_LOG(LS_ERROR) << "Failed to MMALConfigure"; @@ -135,6 +150,7 @@ void MMALV4L2Capturer::OnCaptured(uint8_t* data, uint32_t bytesused) { MMAL_BUFFER_HEADER_T* buffer; while ((buffer = mmal_queue_get(pool_in_->queue)) != nullptr) { + //RTC_LOG(LS_INFO) << "Got input buffer from queue: " << (void*)buffer; buffer->pts = buffer->dts = timestamp_us; buffer->offset = 0; buffer->flags = MMAL_BUFFER_HEADER_FLAG_FRAME; @@ -161,6 +177,7 @@ void MMALV4L2Capturer::MMALInputCallback(MMAL_PORT_T* port, void MMALV4L2Capturer::ResizerFillBuffer() { MMAL_BUFFER_HEADER_T* resizer_out; while ((resizer_out = mmal_queue_get(resizer_pool_out_->queue)) != NULL) { + //RTC_LOG(LS_INFO) << "Got resizer_out from queue: " << (void*)resizer_out; mmal_port_send_buffer(resizer_->output[0], resizer_out); } } From 833a46d63cd10f0dc8bb4a1f5e852ce60f48c5e3 Mon Sep 17 00:00:00 2001 From: melpon Date: Thu, 31 Mar 2022 19:24:33 +0900 Subject: [PATCH 24/70] =?UTF-8?q?`--client-cert`=20=E3=81=A8=20`--client-k?= =?UTF-8?q?ey`=20=E3=81=A7=E3=82=AF=E3=83=A9=E3=82=A4=E3=82=A2=E3=83=B3?= =?UTF-8?q?=E3=83=88=E8=AA=8D=E8=A8=BC=E3=82=92=E3=81=A7=E3=81=8D=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 ++ src/ayame/ayame_client.cpp | 3 ++- src/ayame/ayame_client.h | 2 ++ src/main.cpp | 4 ++++ src/momo_args.h | 2 ++ src/sora/sora_client.cpp | 8 +++++--- src/sora/sora_client.h | 2 ++ src/util.cpp | 13 ++++++++++--- src/websocket.cpp | 23 ++++++++++++++++++++--- src/websocket.h | 6 +++++- 10 files changed, 54 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 33090b9c..d1e2e9c7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,8 @@ ## develop +- [ADD] `--client-cert` と `--client-key` でクライアント認証をできるようにする + - @melpon - [UPDATE] CLI11 を 2.2.0 に上げる - @voluntas - [FIX] Ubuntu 20.04 + H.264 + サイマルキャスト + --hw-mjpeg-decoder true で落ちるのを修正 (#221) diff --git a/src/ayame/ayame_client.cpp b/src/ayame/ayame_client.cpp index 47ff39e3..6c8146dc 100644 --- a/src/ayame/ayame_client.cpp +++ b/src/ayame/ayame_client.cpp @@ -63,7 +63,8 @@ void AyameClient::Reset() { URLParts parts; if (ParseURL(parts)) { - ws_.reset(new Websocket(Websocket::ssl_tag(), ioc_, config_.insecure)); + ws_.reset(new Websocket(Websocket::ssl_tag(), ioc_, config_.insecure, + config_.client_cert, config_.client_key)); } else { ws_.reset(new Websocket(ioc_)); } diff --git a/src/ayame/ayame_client.h b/src/ayame/ayame_client.h index 22ef942f..8bbaca63 100644 --- a/src/ayame/ayame_client.h +++ b/src/ayame/ayame_client.h @@ -23,6 +23,8 @@ struct AyameClientConfig { bool insecure = false; bool no_google_stun = false; + std::string client_cert; + std::string client_key; std::string signaling_url; std::string room_id; diff --git a/src/main.cpp b/src/main.cpp index c267718a..862e8f83 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -259,6 +259,8 @@ int main(int argc, char* argv[]) { config.ignore_disconnect_websocket = args.sora_ignore_disconnect_websocket; config.disconnect_wait_timeout = args.sora_disconnect_wait_timeout; + config.client_cert = args.client_cert; + config.client_key = args.client_key; sora_client = SoraClient::Create(ioc, rtc_manager.get(), std::move(config)); @@ -301,6 +303,8 @@ int main(int argc, char* argv[]) { AyameClientConfig config; config.insecure = args.insecure; config.no_google_stun = args.no_google_stun; + config.client_cert = args.client_cert; + config.client_key = args.client_key; config.signaling_url = args.ayame_signaling_url; config.room_id = args.ayame_room_id; config.client_id = args.ayame_client_id; diff --git a/src/momo_args.h b/src/momo_args.h index dfe7b944..944e62fd 100644 --- a/src/momo_args.h +++ b/src/momo_args.h @@ -39,6 +39,8 @@ struct MomoArgs { bool screen_capture = false; int metrics_port = -1; bool metrics_allow_external_ip = false; + std::string client_cert; + std::string client_key; std::vector sora_signaling_urls; std::string sora_channel_id; diff --git a/src/sora/sora_client.cpp b/src/sora/sora_client.cpp index 6b8c2a32..22f0e1d8 100644 --- a/src/sora/sora_client.cpp +++ b/src/sora/sora_client.cpp @@ -146,7 +146,8 @@ void SoraClient::Connect() { std::shared_ptr ws; if (ssl) { - ws.reset(new Websocket(Websocket::ssl_tag(), ioc_, config_.insecure)); + ws.reset(new Websocket(Websocket::ssl_tag(), ioc_, config_.insecure, + config_.client_cert, config_.client_key)); } else { ws.reset(new Websocket(ioc_)); } @@ -216,8 +217,9 @@ void SoraClient::Redirect(std::string url) { std::shared_ptr ws; if (ssl) { - ws.reset(new Websocket(Websocket::ssl_tag(), self->ioc_, - self->config_.insecure)); + ws.reset(new Websocket( + Websocket::ssl_tag(), self->ioc_, self->config_.insecure, + self->config_.client_cert, self->config_.client_key)); } else { ws.reset(new Websocket(self->ioc_)); } diff --git a/src/sora/sora_client.h b/src/sora/sora_client.h index d69d0278..af03d2d6 100644 --- a/src/sora/sora_client.h +++ b/src/sora/sora_client.h @@ -45,6 +45,8 @@ struct SoraClientConfig { int data_channel_signaling_timeout = 180; boost::optional ignore_disconnect_websocket; int disconnect_wait_timeout = 5; + std::string client_cert; + std::string client_key; }; class SoraClient : public std::enable_shared_from_this, diff --git a/src/util.cpp b/src/util.cpp index fb8eae1d..9ce242dc 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -251,6 +251,13 @@ void Util::ParseArgs(int argc, app.add_flag("--metrics-allow-external-ip", args.metrics_allow_external_ip, "Allow access to Metrics server from external IP"); + app.add_option("--client-cert", args.client_cert, + "Cert file path for client certification (PEM format)") + ->check(CLI::ExistingFile); + app.add_option("--client-key", args.client_key, + "Private key file path for client certification (PEM format)") + ->check(CLI::ExistingFile); + auto test_app = app.add_subcommand( "test", "Mode for momo development with simple HTTP server"); auto ayame_app = app.add_subcommand( @@ -406,11 +413,11 @@ void Util::ParseArgs(int argc, << std::endl; std::cout << std::endl; std::cout << "USE_MMAL_ENCODER=" BOOST_PP_STRINGIZE(USE_MMAL_ENCODER) - << std::endl; + << std::endl; std::cout << "USE_JETSON_ENCODER=" BOOST_PP_STRINGIZE(USE_JETSON_ENCODER) - << std::endl; + << std::endl; std::cout << "USE_NVCODEC_ENCODER=" BOOST_PP_STRINGIZE(USE_NVCODEC_ENCODER) - << std::endl; + << std::endl; std::cout << "USE_SDL2=" BOOST_PP_STRINGIZE(USE_SDL2) << std::endl; exit(0); } diff --git a/src/websocket.cpp b/src/websocket.cpp index 0cbfac24..3dc5637d 100644 --- a/src/websocket.cpp +++ b/src/websocket.cpp @@ -12,7 +12,9 @@ #include "ssl_verifier.h" #include "util.h" -static boost::asio::ssl::context CreateSSLContext() { +static boost::asio::ssl::context CreateSSLContext( + const std::string& client_cert, + const std::string& client_key) { // TLS 1.2 と 1.3 のみ対応 SSL_CTX* handle = ::SSL_CTX_new(::TLS_method()); SSL_CTX_set_min_proto_version(handle, TLS1_2_VERSION); @@ -23,6 +25,16 @@ static boost::asio::ssl::context CreateSSLContext() { boost::asio::ssl::context::no_sslv2 | boost::asio::ssl::context::no_sslv3 | boost::asio::ssl::context::single_dh_use); + if (!client_cert.empty()) { + ctx.use_certificate_file(client_cert, + boost::asio::ssl::context_base::file_format::pem); + RTC_LOG(LS_INFO) << "client_cert=" << client_cert; + } + if (!client_key.empty()) { + ctx.use_private_key_file(client_key, + boost::asio::ssl::context_base::file_format::pem); + RTC_LOG(LS_INFO) << "client_key=" << client_key; + } return ctx; } @@ -34,8 +46,13 @@ Websocket::Websocket(boost::asio::io_context& ioc) } Websocket::Websocket(Websocket::ssl_tag, boost::asio::io_context& ioc, - bool insecure) - : Websocket(Websocket::ssl_tag(), ioc, insecure, CreateSSLContext()) {} + bool insecure, + const std::string& client_cert, + const std::string& client_key) + : Websocket(Websocket::ssl_tag(), + ioc, + insecure, + CreateSSLContext(client_cert, client_key)) {} Websocket::Websocket(Websocket::ssl_tag, boost::asio::io_context& ioc, bool insecure, diff --git a/src/websocket.h b/src/websocket.h index 0d9aa49f..2f455994 100644 --- a/src/websocket.h +++ b/src/websocket.h @@ -44,7 +44,11 @@ class Websocket { // 非SSL+クライアント Websocket(boost::asio::io_context& ioc); // SSL+クライアント - Websocket(ssl_tag, boost::asio::io_context& ioc, bool insecure); + Websocket(ssl_tag, + boost::asio::io_context& ioc, + bool insecure, + const std::string& client_cert, + const std::string& client_key); private: // SSL+クライアントの内部用コンストラクタ From 68a995912669f884d2e3843bcbacc27bde53195f Mon Sep 17 00:00:00 2001 From: melpon Date: Sun, 3 Apr 2022 22:38:46 +0900 Subject: [PATCH 25/70] =?UTF-8?q?Windows=20=E3=81=AE=E3=83=8F=E3=83=BC?= =?UTF-8?q?=E3=83=89=E3=82=A6=E3=82=A7=E3=82=A2=E3=83=87=E3=82=B3=E3=83=BC?= =?UTF-8?q?=E3=83=80=E3=81=AB=E5=AF=BE=E5=BF=9C=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 11 + .github/workflows/daily_build.yml | 7 + .vscode/c_cpp_properties.json | 1 + CMakeLists.txt | 36 ++- VERSION | 1 + build/windows_x86_64/install_deps.ps1 | 32 +++ src/cuda/cuda_context.h | 7 + src/cuda/cuda_context_cuda.cpp | 4 + src/dyn/cuda.h | 65 +++--- src/dyn/dyn.h | 207 +++++++++++------- src/dyn/nvcuvid.h | 60 ++--- src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp | 93 ++++++++ src/hwenc_nvcodec/nvcodec_decoder_cuda.h | 28 +++ .../nvcodec_h264_encoder_cuda.cpp | 4 - .../nvcodec_mjpeg_decoder_cuda.cpp | 38 ---- .../nvcodec_mjpeg_decoder_cuda.h | 20 -- src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp | 4 +- src/hwenc_nvcodec/nvcodec_v4l2_capturer.h | 4 +- src/hwenc_nvcodec/nvcodec_video_decoder.cpp | 127 +++++++++++ src/hwenc_nvcodec/nvcodec_video_decoder.h | 46 ++++ src/main.cpp | 4 +- src/rtc/momo_video_decoder_factory.cpp | 41 +++- src/rtc/momo_video_decoder_factory.h | 14 +- src/rtc/rtc_manager.cpp | 7 +- src/rtc/rtc_manager.h | 2 +- src/video_codec_info.h | 19 ++ 26 files changed, 665 insertions(+), 217 deletions(-) create mode 100644 src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp create mode 100644 src/hwenc_nvcodec/nvcodec_decoder_cuda.h delete mode 100644 src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.cpp delete mode 100644 src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.h create mode 100644 src/hwenc_nvcodec/nvcodec_video_decoder.cpp create mode 100644 src/hwenc_nvcodec/nvcodec_video_decoder.h diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8a57f01a..e9548941 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,13 +29,24 @@ jobs: New-Variable -Name $var[0] -Value $var[1] -Force } echo "::set-output name=boost_version::${BOOST_VERSION}" + echo "::set-output name=wincuda_version::${WINCUDA_VERSION}" id: versions + # Boost はよくダウンロード先に繋がらずエラーになるのでキャッシュする - name: Cache Boost ${{ steps.versions.outputs.boost_version }} id: cache-boost uses: actions/cache@v3 with: path: _cache\boost key: windows-boost-${{ steps.versions.outputs.boost_version }}.zip.v5 + # CUDA のインストールバイナリのダウンロードは 3GB ぐらいあって、 + # ビルドの度に取得するのはつらいのでキャッシュする + # （インストールする nvcc は解凍後で 100MB 程度なのでキャッシュ可能） + - name: Cache NVCC ${{ steps.versions.outputs.wincuda_version }} + id: cache-cuda + uses: actions/cache@v3 + with: + path: build\windows_x86_64\_install\cuda\nvcc + key: windows-cuda-${{ steps.versions.outputs.wincuda_version }}.v1 - run: "& .\\build.ps1 -package" working-directory: build timeout-minutes: 120 diff --git a/.github/workflows/daily_build.yml b/.github/workflows/daily_build.yml index a371e126..a2e8be5e 100644 --- a/.github/workflows/daily_build.yml +++ b/.github/workflows/daily_build.yml @@ -25,6 +25,7 @@ jobs: New-Variable -Name $var[0] -Value $var[1] -Force } echo "::set-output name=boost_version::${BOOST_VERSION}" + echo "::set-output name=wincuda_version::${WINCUDA_VERSION}" id: versions - name: Cache Boost ${{ steps.versions.outputs.boost_version }} id: cache-boost @@ -32,6 +33,12 @@ jobs: with: path: _cache\boost key: windows-boost-${{ steps.versions.outputs.boost_version }}.zip.v5 + - name: Cache NVCC ${{ steps.versions.outputs.wincuda_version }} + id: cache-cuda + uses: actions/cache@v3 + with: + path: build\windows_x86_64\_install\cuda\nvcc + key: windows-cuda-${{ steps.versions.outputs.wincuda_version }}.v1 - run: "& .\\build.ps1 -package" working-directory: build timeout-minutes: 120 diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 3f83baec..42714872 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -14,6 +14,7 @@ "${workspaceFolder}/build/windows_x86_64/_install/boost/include", "${workspaceFolder}/build/windows_x86_64/_install/CLI11/include", "${workspaceFolder}/build/windows_x86_64/_install/SDL2/include/SDL2", + "${workspaceFolder}/build/windows_x86_64/_install/cuda/nvcc/include", "${workspaceFolder}/_build/windows_x86_64" ], "defines": [ diff --git a/CMakeLists.txt b/CMakeLists.txt index 22bbfc14..132eb3c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -502,7 +502,8 @@ if ("${TARGET_OS}" STREQUAL "windows") PRIVATE src/hwenc_nvcodec/nvcodec_h264_encoder.cpp NvCodec/NvCodec/NvEncoder/NvEncoder.cpp - NvCodec/NvCodec/NvEncoder/NvEncoderD3D11.cpp) + NvCodec/NvCodec/NvEncoder/NvEncoderD3D11.cpp + src/hwenc_nvcodec/nvcodec_video_decoder.cpp) target_include_directories(momo PRIVATE NvCodec/include @@ -512,6 +513,39 @@ if ("${TARGET_OS}" STREQUAL "windows") DXGI.lib D3D11.lib ) + + # enable_language(CUDA) は CUDA 用 Visual Studio Integration を利用して CUDA を検出しているが、 + # CUDA ドライバがインストールできない場所では VS Integration をインストールできないため、 + # enable_language(CUDA) は利用できない。 + # なので（deprecated だけど）FindCUDA を利用してコンパイルする。 + + set(CUDA_TOOLKIT_ROOT_DIR ${_INSTALL_DIR}/cuda/nvcc) + find_package(CUDA REQUIRED) + + set_source_files_properties( + src/cuda/cuda_context_cuda.cpp + NvCodec/NvCodec/NvDecoder/NvDecoder.cpp + src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp + PROPERTIES + CUDA_SOURCE_PROPERTY_FORMAT OBJ + ) + cuda_compile(CUDA_FILES + src/cuda/cuda_context_cuda.cpp + NvCodec/NvCodec/NvDecoder/NvDecoder.cpp + src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp + OPTIONS + -Xcompiler /utf-8 + -Xcompiler /I${CMAKE_CURRENT_SOURCE_DIR}/NvCodec/include + -Xcompiler /I${CMAKE_CURRENT_SOURCE_DIR}/NvCodec/NvCodec + -Xcompiler /I${CMAKE_CURRENT_SOURCE_DIR}/src + ) + target_sources(momo PRIVATE ${CUDA_FILES}) + target_include_directories(momo PRIVATE ${CUDA_INCLUDE_DIRS}) + target_link_libraries(momo + PRIVATE + ${CUDA_TOOLKIT_ROOT_DIR}/lib/x64/cudart_static.lib + #${CUDA_LIBRARIES} + ) endif() elseif (TARGET_OS STREQUAL "macos") target_sources(momo diff --git a/VERSION b/VERSION index 2ea3eb07..dd5d3948 100644 --- a/VERSION +++ b/VERSION @@ -5,3 +5,4 @@ CLI11_VERSION=2.2.0 SDL2_VERSION=2.0.20 CMAKE_VERSION=3.22.3 CUDA_VERSION=11.0.2-1 +WINCUDA_VERSION=10.2 \ No newline at end of file diff --git a/build/windows_x86_64/install_deps.ps1 b/build/windows_x86_64/install_deps.ps1 index 178b5be1..822c1c76 100644 --- a/build/windows_x86_64/install_deps.ps1 +++ b/build/windows_x86_64/install_deps.ps1 @@ -134,3 +134,35 @@ if (!(Test-Path "$INSTALL_DIR\CLI11\include\CLI\CLI.hpp")) { Move-Item CLI11-${CLI11_VERSION} CLI11 Pop-Location } + +if (!(Test-Path "$INSTALL_DIR\cuda\nvcc")) { + if ("$WINCUDA_VERSION" -eq "10.2") { + $_URL = "http://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda_10.2.89_441.22_win10.exe" + $_FILE = "$SOURCE_DIR\cuda_10.2.89_441.22_win10.exe" + } elseif ("$WINCUDA_VERSION" -eq "11.0.2") { + $_URL = "https://developer.download.nvidia.com/compute/cuda/11.0.2/local_installers/cuda_11.0.2_451.48_win10.exe" + $_FILE = "$SOURCE_DIR\cuda_11.0.2_451.48_win10.exe" + } else { + # バージョンが増えたらこの分岐を増やしていく + throw "CUDA-$WINCUDA_VERSION URL not specified" + } + + Push-Location $SOURCE_DIR + if (!(Test-Path $_FILE)) { + Invoke-WebRequest -Uri $_URL -OutFile $_FILE + } + Pop-Location + if (Test-Path "$BUILD_DIR\cuda") { + Remove-Item "$BUILD_DIR\cuda" -Recurse -Force + } + mkdir $BUILD_DIR\cuda -Force + Push-Location $BUILD_DIR\cuda + # サイレントインストールとかせずに、単に展開だけして nvcc を利用する + 7z x $_FILE + if (Test-Path "$INSTALL_DIR\cuda") { + Remove-Item $INSTALL_DIR\cuda -Recurse -Force + } + mkdir $INSTALL_DIR\cuda + Move-Item nvcc $INSTALL_DIR\cuda\nvcc + Pop-Location +} \ No newline at end of file diff --git a/src/cuda/cuda_context.h b/src/cuda/cuda_context.h index 4c11a9f5..a136a3a5 100644 --- a/src/cuda/cuda_context.h +++ b/src/cuda/cuda_context.h @@ -13,4 +13,11 @@ class CudaContext { std::shared_ptr impl_; }; +enum class CudaVideoCodec { + H264, + VP8, + VP9, + JPEG, +}; + #endif \ No newline at end of file diff --git a/src/cuda/cuda_context_cuda.cpp b/src/cuda/cuda_context_cuda.cpp index c36dca55..309cbdd0 100644 --- a/src/cuda/cuda_context_cuda.cpp +++ b/src/cuda/cuda_context_cuda.cpp @@ -4,6 +4,10 @@ #include "dyn/cuda.h" +// どこかにグローバルな logger の定義が必要 +simplelogger::Logger* logger = + simplelogger::LoggerFactory::CreateConsoleLogger(); + CUcontext GetCudaContext(std::shared_ptr ctx) { return static_cast(ctx->Context()); } diff --git a/src/dyn/cuda.h b/src/dyn/cuda.h index f512076a..d2798501 100644 --- a/src/dyn/cuda.h +++ b/src/dyn/cuda.h @@ -1,31 +1,34 @@ -#ifndef DYN_CUDA_H_ -#define DYN_CUDA_H_ - -// CUDA -#include - -#include "dyn.h" - -namespace dyn { - -static const char CUDA_SO[] = "libcuda.so.1"; -DYN_REGISTER(CUDA_SO, cuInit); -DYN_REGISTER(CUDA_SO, cuDeviceGet); -DYN_REGISTER(CUDA_SO, cuDeviceGetCount); -DYN_REGISTER(CUDA_SO, cuDeviceGetName); -DYN_REGISTER(CUDA_SO, cuCtxCreate); -DYN_REGISTER(CUDA_SO, cuCtxDestroy); -DYN_REGISTER(CUDA_SO, cuCtxPushCurrent); -DYN_REGISTER(CUDA_SO, cuCtxPopCurrent); -DYN_REGISTER(CUDA_SO, cuGetErrorName); -DYN_REGISTER(CUDA_SO, cuMemAlloc); -DYN_REGISTER(CUDA_SO, cuMemAllocPitch); -DYN_REGISTER(CUDA_SO, cuMemFree); -DYN_REGISTER(CUDA_SO, cuMemcpy2D); -DYN_REGISTER(CUDA_SO, cuMemcpy2DAsync); -DYN_REGISTER(CUDA_SO, cuMemcpy2DUnaligned); -DYN_REGISTER(CUDA_SO, cuStreamSynchronize); - -} // namespace dyn - -#endif // DYN_CUDA_H_ +#ifndef DYN_CUDA_H_ +#define DYN_CUDA_H_ + +#include + +#include "dyn.h" + +namespace dyn { + +#if defined(WIN32) +static const char CUDA_SO[] = "nvcuda.dll"; +#else +static const char CUDA_SO[] = "libcuda.so.1"; +#endif +DYN_REGISTER(CUDA_SO, cuInit); +DYN_REGISTER(CUDA_SO, cuDeviceGet); +DYN_REGISTER(CUDA_SO, cuDeviceGetCount); +DYN_REGISTER(CUDA_SO, cuDeviceGetName); +DYN_REGISTER(CUDA_SO, cuCtxCreate); +DYN_REGISTER(CUDA_SO, cuCtxDestroy); +DYN_REGISTER(CUDA_SO, cuCtxPushCurrent); +DYN_REGISTER(CUDA_SO, cuCtxPopCurrent); +DYN_REGISTER(CUDA_SO, cuGetErrorName); +DYN_REGISTER(CUDA_SO, cuMemAlloc); +DYN_REGISTER(CUDA_SO, cuMemAllocPitch); +DYN_REGISTER(CUDA_SO, cuMemFree); +DYN_REGISTER(CUDA_SO, cuMemcpy2D); +DYN_REGISTER(CUDA_SO, cuMemcpy2DAsync); +DYN_REGISTER(CUDA_SO, cuMemcpy2DUnaligned); +DYN_REGISTER(CUDA_SO, cuStreamSynchronize); + +} // namespace dyn + +#endif // DYN_CUDA_H_ \ No newline at end of file diff --git a/src/dyn/dyn.h b/src/dyn/dyn.h index 48bb0187..0b578c70 100644 --- a/src/dyn/dyn.h +++ b/src/dyn/dyn.h @@ -1,82 +1,125 @@ -#ifndef DYN_DYN_H_ -#define DYN_DYN_H_ - -#include -#include -#include - -// Linux -#include - -namespace dyn { -class DynModule { - public: - static DynModule& Instance() { - static DynModule instance; - return instance; - } - - bool IsLoadable(const char* name) { - void* module = dlopen(name, RTLD_LAZY); - if (module == nullptr) { - return false; - } - dlclose(module); - return true; - } - - void* Get(const char* name) { - auto it = modules_.find(name); - if (it != modules_.end()) { - return it->second.get(); - } - void* module = dlopen(name, RTLD_LAZY); - if (module == nullptr) { - return nullptr; - } - modules_.insert(decltype(modules_)::value_type( - name, std::unique_ptr(module))); - return module; - } - - private: - struct dlcloser { - void operator()(void* p) { - if (p != nullptr) { - ::dlclose(p); - } - } - }; - - std::map> modules_; -}; -} // namespace dyn - -// text の定義を全て展開した上で文字列化する。 -// 単純に #text とした場合、全て展開する前に文字列化されてしまう - -#if defined(_WIN32) -#define DYN_STRINGIZE(text) DYN_STRINGIZE_((text)) -#define DYN_STRINGIZE_(x) DYN_STRINGIZE_I x -#else -#define DYN_STRINGIZE(text) DYN_STRINGIZE_I(text) -#endif - -#define DYN_STRINGIZE_I(text) #text - -#define DYN_REGISTER(soname, func) \ - template \ - inline auto func(Args... args) { \ - typedef std::add_pointer::type func_type; \ - void* module = DynModule::Instance().Get(soname); \ - if (module == nullptr) { \ - exit(1); \ - } \ - auto f = (func_type)dlsym(module, DYN_STRINGIZE(func)); \ - if (f == nullptr) { \ - exit(1); \ - } \ - return f(args...); \ - } - -#endif // DYN_DYN_H_ +#ifndef DYN_DYN_H_ +#define DYN_DYN_H_ + +#include +#include +#include + +#if defined(_WIN32) +#include +#elif defined(__linux__) +// Linux +#include +#endif + +namespace dyn { +class DynModule { + public: + static DynModule& Instance() { + static DynModule instance; + return instance; + } +#if defined(_WIN32) + typedef HINSTANCE__ module_t; + typedef HMODULE module_ptr_t; +#else + typedef void module_t; + typedef void* module_ptr_t; +#endif + + bool IsLoadable(const char* name) { +#if defined(_WIN32) + module_ptr_t module = LoadLibraryA(name); + if (module == nullptr) { + return false; + } + FreeLibrary(module); + return true; +#elif defined(__linux__) + module_ptr_t module = dlopen(name, RTLD_LAZY); + if (module == nullptr) { + return false; + } + dlclose(module); + return true; +#else + return false; +#endif + } + + module_ptr_t Get(const char* name) { + auto it = modules_.find(name); + if (it != modules_.end()) { + return it->second.get(); + } +#if defined(_WIN32) + module_ptr_t module = LoadLibraryA(name); +#elif defined(__linux__) + module_ptr_t module = dlopen(name, RTLD_LAZY); +#else + module_ptr_t module = nullptr; +#endif + if (module == nullptr) { + return nullptr; + } + modules_.insert(decltype(modules_)::value_type( + name, std::unique_ptr(module))); + return module; + } + + void* GetFunc(const char* soname, const char* name) { + module_ptr_t module = Get(soname); + if (module == nullptr) { + return nullptr; + } +#if defined(_WIN32) + return ::GetProcAddress(module, name); +#elif defined(__linux__) + return dlsym(module, name); +#else + return nullptr; +#endif + } + + private: + struct dlcloser { + void operator()(module_ptr_t p) { + if (p != nullptr) { +#if defined(_WIN32) + FreeLibrary(p); +#elif defined(__linux__) + ::dlclose(p); +#else +#endif + } + } + }; + + std::map> modules_; +}; +} // namespace dyn + +// text の定義を全て展開した上で文字列化する。 +// 単純に #text とした場合、全て展開する前に文字列化されてしまう +#define DYN_STRINGIZE(text) DYN_STRINGIZE_((text)) + +#if defined(_WIN32) +#define DYN_STRINGIZE_(x) DYN_STRINGIZE_I x +#else +#define DYN_STRINGIZE_(x) DYN_STRINGIZE_I ## x +#endif + +#define DYN_STRINGIZE_I(text) #text + +#define DYN_REGISTER(soname, func) \ + template \ + inline auto func(Args... args) { \ + typedef std::add_pointer::type func_type; \ + auto f = (func_type)DynModule::Instance().GetFunc(soname, DYN_STRINGIZE(func)); \ + if (f == nullptr) { \ + exit(1); \ + } \ + return f(args...); \ + } + +#endif // DYN_DYN_H_ \ No newline at end of file diff --git a/src/dyn/nvcuvid.h b/src/dyn/nvcuvid.h index 4200078d..7cb33fca 100644 --- a/src/dyn/nvcuvid.h +++ b/src/dyn/nvcuvid.h @@ -1,28 +1,32 @@ -#ifndef DYN_NVCUVID_H_ -#define DYN_NVCUVID_H_ - -#include "dyn.h" - -// CUDA -#include - -namespace dyn { - -static const char NVCUVID_SO[] = "libnvcuvid.so.1"; -DYN_REGISTER(NVCUVID_SO, cuvidCreateDecoder); -DYN_REGISTER(NVCUVID_SO, cuvidReconfigureDecoder); -DYN_REGISTER(NVCUVID_SO, cuvidDestroyDecoder); -DYN_REGISTER(NVCUVID_SO, cuvidDecodePicture); -DYN_REGISTER(NVCUVID_SO, cuvidGetDecodeStatus); -DYN_REGISTER(NVCUVID_SO, cuvidGetDecoderCaps); -DYN_REGISTER(NVCUVID_SO, cuvidCreateVideoParser); -DYN_REGISTER(NVCUVID_SO, cuvidDestroyVideoParser); -DYN_REGISTER(NVCUVID_SO, cuvidParseVideoData); -DYN_REGISTER(NVCUVID_SO, cuvidMapVideoFrame); -DYN_REGISTER(NVCUVID_SO, cuvidUnmapVideoFrame); -DYN_REGISTER(NVCUVID_SO, cuvidCtxLockCreate); -DYN_REGISTER(NVCUVID_SO, cuvidCtxLockDestroy); - -} // namespace dyn - -#endif // DYN_NVCUVID_H_ +#ifndef DYN_NVCUVID_H_ +#define DYN_NVCUVID_H_ + +#include "dyn.h" + +// defs +#include + +namespace dyn { + +#if defined(WIN32) +static const char NVCUVID_SO[] = "nvcuvid.dll"; +#else +static const char NVCUVID_SO[] = "libnvcuvid.so.1"; +#endif +DYN_REGISTER(NVCUVID_SO, cuvidCreateDecoder); +DYN_REGISTER(NVCUVID_SO, cuvidReconfigureDecoder); +DYN_REGISTER(NVCUVID_SO, cuvidDestroyDecoder); +DYN_REGISTER(NVCUVID_SO, cuvidDecodePicture); +DYN_REGISTER(NVCUVID_SO, cuvidGetDecodeStatus); +DYN_REGISTER(NVCUVID_SO, cuvidGetDecoderCaps); +DYN_REGISTER(NVCUVID_SO, cuvidCreateVideoParser); +DYN_REGISTER(NVCUVID_SO, cuvidDestroyVideoParser); +DYN_REGISTER(NVCUVID_SO, cuvidParseVideoData); +DYN_REGISTER(NVCUVID_SO, cuvidMapVideoFrame); +DYN_REGISTER(NVCUVID_SO, cuvidUnmapVideoFrame); +DYN_REGISTER(NVCUVID_SO, cuvidCtxLockCreate); +DYN_REGISTER(NVCUVID_SO, cuvidCtxLockDestroy); + +} // namespace dyn + +#endif // DYN_NVCUVID_H_ \ No newline at end of file diff --git a/src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp b/src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp new file mode 100644 index 00000000..4eb1195a --- /dev/null +++ b/src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp @@ -0,0 +1,93 @@ +#include "nvcodec_decoder_cuda.h" + +// NvCodec +#include + +#include "cuda/cuda_context_cuda.h" + +static cudaVideoCodec ToCudaVideoCodec(CudaVideoCodec codec) { + return codec == CudaVideoCodec::H264 ? cudaVideoCodec_H264 + : codec == CudaVideoCodec::VP8 ? cudaVideoCodec_VP8 + : codec == CudaVideoCodec::VP9 ? cudaVideoCodec_VP9 + : cudaVideoCodec_JPEG; +} + +#define CUDA_DRVAPI_CALL(call) \ + do { \ + CUresult err__ = call; \ + if (err__ != CUDA_SUCCESS) { \ + const char* szErrName = NULL; \ + dyn::cuGetErrorName(err__, &szErrName); \ + std::ostringstream errorLog; \ + errorLog << "CUDA driver API error " << szErrName; \ + throw NVDECException::makeNVDECException( \ + errorLog.str(), err__, __FUNCTION__, __FILE__, __LINE__); \ + } \ + } while (0) + +NvCodecDecoderCuda::NvCodecDecoderCuda(std::shared_ptr ctx, + CudaVideoCodec codec) + : impl_(new NvDecoder(GetCudaContext(ctx), + false, + ToCudaVideoCodec(codec), + false, + true, + nullptr, + nullptr, + 3840, + 3840)) { + // このコーデックでデコード可能かどうかを調べる + CUVIDDECODECAPS decodecaps; + memset(&decodecaps, 0, sizeof(decodecaps)); + + decodecaps.eCodecType = ToCudaVideoCodec(codec); + decodecaps.eChromaFormat = cudaVideoChromaFormat_420; + decodecaps.nBitDepthMinus8 = 0; + + CUDA_DRVAPI_CALL(dyn::cuCtxPushCurrent(GetCudaContext(ctx))); + NVDEC_API_CALL(dyn::cuvidGetDecoderCaps(&decodecaps)); + CUDA_DRVAPI_CALL(dyn::cuCtxPopCurrent(nullptr)); + + if (!decodecaps.bIsSupported) { + throw NVDECException::makeNVDECException( + "Specified video codec not supported", CUDA_ERROR_NOT_SUPPORTED, + __FUNCTION__, __FILE__, __LINE__); + } +} + +static NvDecoder* GetDecoder(std::shared_ptr impl) { + return std::static_pointer_cast(impl).get(); +} + +int NvCodecDecoderCuda::Decode(const uint8_t* pData, int nSize) { + return GetDecoder(impl_)->Decode(pData, nSize); +} + +uint8_t* NvCodecDecoderCuda::GetFrame() { + return GetDecoder(impl_)->GetFrame(); +} + +uint8_t* NvCodecDecoderCuda::GetLockedFrame() { + return GetDecoder(impl_)->GetLockedFrame(); +} + +void NvCodecDecoderCuda::UnlockFrame(uint8_t* pFrame) { + GetDecoder(impl_)->UnlockFrame(&pFrame); +} + +std::string NvCodecDecoderCuda::GetVideoInfo() const { + return GetDecoder(impl_)->GetVideoInfo(); +} +int NvCodecDecoderCuda::GetWidth() const { + return GetDecoder(impl_)->GetWidth(); +} +int NvCodecDecoderCuda::GetHeight() const { + return GetDecoder(impl_)->GetHeight(); +} +int NvCodecDecoderCuda::GetDeviceFramePitch() const { + return GetDecoder(impl_)->GetDeviceFramePitch(); +} + +int NvCodecDecoderCuda::setReconfigParams() { + return GetDecoder(impl_)->setReconfigParams(nullptr, nullptr); +} \ No newline at end of file diff --git a/src/hwenc_nvcodec/nvcodec_decoder_cuda.h b/src/hwenc_nvcodec/nvcodec_decoder_cuda.h new file mode 100644 index 00000000..1b70817e --- /dev/null +++ b/src/hwenc_nvcodec/nvcodec_decoder_cuda.h @@ -0,0 +1,28 @@ +#ifndef NVCODEC_DECODER_CUDA_H_ +#define NVCODEC_DECODER_CUDA_H_ + +#include +#include + +#include "cuda/cuda_context.h" + +class NvCodecDecoderCuda { + public: + NvCodecDecoderCuda(std::shared_ptr ctx, CudaVideoCodec codec); + int Decode(const uint8_t* pData, int nSize); + uint8_t* GetFrame(); + uint8_t* GetLockedFrame(); + void UnlockFrame(uint8_t* pFrame); + + std::string GetVideoInfo() const; + int GetWidth() const; + int GetHeight() const; + int GetDeviceFramePitch() const; + + int setReconfigParams(); + + private: + std::shared_ptr impl_; +}; + +#endif \ No newline at end of file diff --git a/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp b/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp index 28b26197..662a54f4 100644 --- a/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp +++ b/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp @@ -9,10 +9,6 @@ #include "cuda/cuda_context_cuda.h" #include "dyn/cuda.h" -// どこかにグローバルな logger の定義が必要 -simplelogger::Logger* logger = - simplelogger::LoggerFactory::CreateConsoleLogger(); - class NvCodecH264EncoderCudaImpl { public: NvCodecH264EncoderCudaImpl(std::shared_ptr ctx); diff --git a/src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.cpp b/src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.cpp deleted file mode 100644 index 159e3397..00000000 --- a/src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include "nvcodec_mjpeg_decoder_cuda.h" - -// NvCodec -#include - -#include "cuda/cuda_context_cuda.h" - -NvCodecMjpegDecoderCuda::NvCodecMjpegDecoderCuda( - std::shared_ptr ctx) - : impl_(new NvDecoder(GetCudaContext(ctx), - false, - cudaVideoCodec_JPEG, - false, - true, - nullptr, - nullptr, - 3840, - 2160)) {} - -static NvDecoder* GetDecoder(std::shared_ptr impl) { - return std::static_pointer_cast(impl).get(); -} - -int NvCodecMjpegDecoderCuda::Decode(const uint8_t* pData, int nSize) { - return GetDecoder(impl_)->Decode(pData, nSize); -} - -uint8_t* NvCodecMjpegDecoderCuda::GetFrame() { - return GetDecoder(impl_)->GetFrame(); -} - -uint8_t* NvCodecMjpegDecoderCuda::GetLockedFrame() { - return GetDecoder(impl_)->GetLockedFrame(); -} - -void NvCodecMjpegDecoderCuda::UnlockFrame(uint8_t* pFrame) { - return GetDecoder(impl_)->UnlockFrame(&pFrame); -} \ No newline at end of file diff --git a/src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.h b/src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.h deleted file mode 100644 index fd154ee1..00000000 --- a/src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef NVCODEC_MJPEG_DECODER_CUDA_H_ -#define NVCODEC_MJPEG_DECODER_CUDA_H_ - -#include - -#include "cuda/cuda_context.h" - -class NvCodecMjpegDecoderCuda { - public: - NvCodecMjpegDecoderCuda(std::shared_ptr ctx); - int Decode(const uint8_t* pData, int nSize); - uint8_t* GetFrame(); - uint8_t* GetLockedFrame(); - void UnlockFrame(uint8_t* pFrame); - - private: - std::shared_ptr impl_; -}; - -#endif // NVCODEC_MJPEG_DECODER_CUDA_H_ \ No newline at end of file diff --git a/src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp b/src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp index 1c48e4c3..b84a197c 100644 --- a/src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp +++ b/src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp @@ -52,8 +52,8 @@ rtc::scoped_refptr NvCodecV4L2Capturer::Create( rtc::scoped_refptr v4l2_capturer( new rtc::RefCountedObject()); - v4l2_capturer->decoder_.reset( - new NvCodecMjpegDecoderCuda(config.cuda_context)); + v4l2_capturer->decoder_.reset(new NvCodecDecoderCuda( + config.cuda_context, NvCodecDecoderCuda::VideoCodec::JPEG)); if (v4l2_capturer->Init((const char*)&unique_name, config.video_device) < 0) { RTC_LOG(LS_WARNING) << "Failed to create NvCodecV4L2Capturer(" diff --git a/src/hwenc_nvcodec/nvcodec_v4l2_capturer.h b/src/hwenc_nvcodec/nvcodec_v4l2_capturer.h index a518031c..0dbc6afc 100644 --- a/src/hwenc_nvcodec/nvcodec_v4l2_capturer.h +++ b/src/hwenc_nvcodec/nvcodec_v4l2_capturer.h @@ -3,7 +3,7 @@ #include "v4l2_video_capturer/v4l2_video_capturer.h" -#include "./nvcodec_mjpeg_decoder_cuda.h" +#include "./nvcodec_decoder_cuda.h" struct NvCodecV4L2CapturerConfig : V4L2VideoCapturerConfig { NvCodecV4L2CapturerConfig(const V4L2VideoCapturerConfig& config) { @@ -27,7 +27,7 @@ class NvCodecV4L2Capturer : public V4L2VideoCapturer { void OnCaptured(uint8_t* data, uint32_t bytesused) override; - std::shared_ptr decoder_; + std::shared_ptr decoder_; }; #endif // NVCODEC_V4L2_CAPTURER_H_ diff --git a/src/hwenc_nvcodec/nvcodec_video_decoder.cpp b/src/hwenc_nvcodec/nvcodec_video_decoder.cpp new file mode 100644 index 00000000..f4b08129 --- /dev/null +++ b/src/hwenc_nvcodec/nvcodec_video_decoder.cpp @@ -0,0 +1,127 @@ +#include "nvcodec_video_decoder.h" + +// WebRTC +#include +#include +#include +#include +#include + +#include "dyn/cuda.h" +#include "dyn/nvcuvid.h" + +NvCodecVideoDecoder::NvCodecVideoDecoder(std::shared_ptr ctx, + CudaVideoCodec codec) + : context_(ctx), + codec_(codec), + decode_complete_callback_(nullptr), + buffer_pool_(false, 300 /* max_number_of_buffers*/) {} + +NvCodecVideoDecoder::~NvCodecVideoDecoder() { + Release(); +} + +bool NvCodecVideoDecoder::IsSupported(CudaVideoCodec codec) { + // CUDA 周りのライブラリがロードできるか確認する + if (!dyn::DynModule::Instance().IsLoadable(dyn::CUDA_SO)) { + return false; + } + if (!dyn::DynModule::Instance().IsLoadable(dyn::NVCUVID_SO)) { + return false; + } + + try { + auto context = CudaContext::Create(); + auto p = new NvCodecDecoderCuda(context, codec); + delete p; + return true; + } catch (...) { + return false; + } +} + +bool NvCodecVideoDecoder::Configure(const Settings& settings) { + return InitNvCodec(); +} + +int32_t NvCodecVideoDecoder::Decode(const webrtc::EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) { + if (decoder_ == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (decode_complete_callback_ == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (input_image.data() == nullptr && input_image.size() > 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + uint8_t** frames = nullptr; + int frame_count = + decoder_->Decode(input_image.data(), (int)input_image.size()); + if (frame_count == 0) { + return WEBRTC_VIDEO_CODEC_OK; + } + + // 初回だけデコード情報を出力する + if (!output_info_) { + RTC_LOG(LS_INFO) << decoder_->GetVideoInfo(); + output_info_ = true; + } + + uint32_t pts = input_image.Timestamp(); + + for (int i = 0; i < frame_count; i++) { + const auto* frame = decoder_->GetFrame(); + // NV12 から I420 に変換 + rtc::scoped_refptr i420_buffer = + buffer_pool_.CreateI420Buffer(decoder_->GetWidth(), + decoder_->GetHeight()); + libyuv::NV12ToI420( + frame, decoder_->GetDeviceFramePitch(), + frame + decoder_->GetHeight() * decoder_->GetDeviceFramePitch(), + decoder_->GetDeviceFramePitch(), i420_buffer->MutableDataY(), + i420_buffer->StrideY(), i420_buffer->MutableDataU(), + i420_buffer->StrideU(), i420_buffer->MutableDataV(), + i420_buffer->StrideV(), decoder_->GetWidth(), decoder_->GetHeight()); + + webrtc::VideoFrame decoded_image = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(i420_buffer) + .set_timestamp_rtp(pts) + .build(); + decode_complete_callback_->Decoded(decoded_image, absl::nullopt, + absl::nullopt); + + // 次のフレームで縦横サイズが変わったときに追従するためのマジックコード + decoder_->setReconfigParams(); + } + + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t NvCodecVideoDecoder::RegisterDecodeCompleteCallback( + webrtc::DecodedImageCallback* callback) { + decode_complete_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t NvCodecVideoDecoder::Release() { + ReleaseNvCodec(); + buffer_pool_.Release(); + return WEBRTC_VIDEO_CODEC_OK; +} + +const char* NvCodecVideoDecoder::ImplementationName() const { + return "NVIDIA VIDEO CODEC SDK"; +} + +bool NvCodecVideoDecoder::InitNvCodec() { + decoder_.reset(new NvCodecDecoderCuda(context_, codec_)); + output_info_ = false; + return true; +} + +void NvCodecVideoDecoder::ReleaseNvCodec() { + decoder_.reset(); +} diff --git a/src/hwenc_nvcodec/nvcodec_video_decoder.h b/src/hwenc_nvcodec/nvcodec_video_decoder.h new file mode 100644 index 00000000..e7ce883f --- /dev/null +++ b/src/hwenc_nvcodec/nvcodec_video_decoder.h @@ -0,0 +1,46 @@ +#ifndef NVCODEC_VIDEO_DECODER_H_ +#define NVCODEC_VIDEO_DECODER_H_ + +// WebRTC +#include +#include +#include + +#include "cuda/cuda_context.h" +#include "nvcodec_decoder_cuda.h" + +class NvCodecVideoDecoder : public webrtc::VideoDecoder { + public: + NvCodecVideoDecoder(std::shared_ptr context, + CudaVideoCodec codec); + ~NvCodecVideoDecoder() override; + + static bool IsSupported(CudaVideoCodec codec); + + bool Configure(const Settings& settings) override; + + int32_t Decode(const webrtc::EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) override; + + int32_t RegisterDecodeCompleteCallback( + webrtc::DecodedImageCallback* callback) override; + + int32_t Release() override; + + const char* ImplementationName() const override; + + private: + bool InitNvCodec(); + void ReleaseNvCodec(); + + webrtc::DecodedImageCallback* decode_complete_callback_ = nullptr; + webrtc::VideoFrameBufferPool buffer_pool_; + + std::shared_ptr context_; + CudaVideoCodec codec_; + std::unique_ptr decoder_; + bool output_info_ = false; +}; + +#endif // NVCODEC_VIDEO_DECODER_H_ diff --git a/src/main.cpp b/src/main.cpp index 862e8f83..17d409b5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -86,7 +86,7 @@ int main(int argc, char* argv[]) { } rtc::LogMessage::AddLogToStream(log_sink.get(), rtc::LS_INFO); -#if defined(__linux__) && USE_NVCODEC_ENCODER +#if USE_NVCODEC_ENCODER auto cuda_context = CudaContext::Create(); #endif @@ -190,7 +190,7 @@ int main(int argc, char* argv[]) { rtcm_config.priority = args.priority; -#if defined(__linux__) && USE_NVCODEC_ENCODER +#if USE_NVCODEC_ENCODER rtcm_config.cuda_context = cuda_context; #endif diff --git a/src/rtc/momo_video_decoder_factory.cpp b/src/rtc/momo_video_decoder_factory.cpp index ce45d047..517d80cb 100644 --- a/src/rtc/momo_video_decoder_factory.cpp +++ b/src/rtc/momo_video_decoder_factory.cpp @@ -44,11 +44,21 @@ MomoVideoDecoderFactory::MomoVideoDecoderFactory( VideoCodecInfo::Type vp8_decoder, VideoCodecInfo::Type vp9_decoder, VideoCodecInfo::Type av1_decoder, - VideoCodecInfo::Type h264_decoder) + VideoCodecInfo::Type h264_decoder +#if USE_NVCODEC_ENCODER + , + std::shared_ptr cuda_context +#endif + ) : vp8_decoder_(vp8_decoder), vp9_decoder_(vp9_decoder), av1_decoder_(av1_decoder), - h264_decoder_(h264_decoder) { + h264_decoder_(h264_decoder) +#if USE_NVCODEC_ENCODER + , + cuda_context_(cuda_context) +#endif +{ #if defined(__APPLE__) video_decoder_factory_ = CreateObjCDecoderFactory(); #endif @@ -59,13 +69,15 @@ MomoVideoDecoderFactory::GetSupportedFormats() const { // VP8 if (vp8_decoder_ == VideoCodecInfo::Type::Software || - vp9_decoder_ == VideoCodecInfo::Type::Jetson) { + vp8_decoder_ == VideoCodecInfo::Type::Jetson || + vp8_decoder_ == VideoCodecInfo::Type::NVIDIA) { supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName)); } // VP9 if (vp9_decoder_ == VideoCodecInfo::Type::Software || - vp9_decoder_ == VideoCodecInfo::Type::Jetson) { + vp9_decoder_ == VideoCodecInfo::Type::Jetson || + vp9_decoder_ == VideoCodecInfo::Type::NVIDIA) { for (const webrtc::SdpVideoFormat& format : webrtc::SupportedVP9Codecs()) { supported_codecs.push_back(format); } @@ -114,6 +126,13 @@ MomoVideoDecoderFactory::CreateVideoDecoder( } if (absl::EqualsIgnoreCase(format.name, cricket::kVp8CodecName)) { +#if USE_NVCODEC_ENCODER + if (vp8_decoder_ == VideoCodecInfo::Type::NVIDIA) { + return std::unique_ptr( + absl::make_unique(cuda_context_, + CudaVideoCodec::VP8)); + } +#endif #if USE_JETSON_ENCODER if (vp8_decoder_ == VideoCodecInfo::Type::Jetson) { return std::unique_ptr( @@ -127,6 +146,13 @@ MomoVideoDecoderFactory::CreateVideoDecoder( } if (absl::EqualsIgnoreCase(format.name, cricket::kVp9CodecName)) { +#if USE_NVCODEC_ENCODER + if (vp9_decoder_ == VideoCodecInfo::Type::NVIDIA) { + return std::unique_ptr( + absl::make_unique(cuda_context_, + CudaVideoCodec::VP9)); + } +#endif #if USE_JETSON_ENCODER if (vp9_decoder_ == VideoCodecInfo::Type::Jetson) { return std::unique_ptr( @@ -154,6 +180,13 @@ MomoVideoDecoderFactory::CreateVideoDecoder( } #endif +#if USE_NVCODEC_ENCODER + if (h264_decoder_ == VideoCodecInfo::Type::NVIDIA) { + return std::unique_ptr( + absl::make_unique(cuda_context_, + CudaVideoCodec::H264)); + } +#endif #if USE_JETSON_ENCODER if (h264_decoder_ == VideoCodecInfo::Type::Jetson) { return std::unique_ptr( diff --git a/src/rtc/momo_video_decoder_factory.h b/src/rtc/momo_video_decoder_factory.h index bd60e1d5..b0c5ed59 100644 --- a/src/rtc/momo_video_decoder_factory.h +++ b/src/rtc/momo_video_decoder_factory.h @@ -9,6 +9,10 @@ #include "video_codec_info.h" +#if USE_NVCODEC_ENCODER +#include "cuda/cuda_context.h" +#endif + class MomoVideoDecoderFactory : public webrtc::VideoDecoderFactory { VideoCodecInfo::Type vp8_decoder_; VideoCodecInfo::Type vp9_decoder_; @@ -20,13 +24,21 @@ class MomoVideoDecoderFactory : public webrtc::VideoDecoderFactory { MomoVideoDecoderFactory(VideoCodecInfo::Type vp8_decoder, VideoCodecInfo::Type vp9_decoder, VideoCodecInfo::Type av1_decoder, - VideoCodecInfo::Type h264_decoder); + VideoCodecInfo::Type h264_decoder +#if USE_NVCODEC_ENCODER + , + std::shared_ptr cuda_context +#endif + ); virtual ~MomoVideoDecoderFactory() {} std::vector GetSupportedFormats() const override; std::unique_ptr CreateVideoDecoder( const webrtc::SdpVideoFormat& format) override; +#if USE_NVCODEC_ENCODER + std::shared_ptr cuda_context_; +#endif }; #endif // MOMO_VIDEO_DECODER_FACTORY_H_ diff --git a/src/rtc/rtc_manager.cpp b/src/rtc/rtc_manager.cpp index 24e165bf..38d4f3ff 100644 --- a/src/rtc/rtc_manager.cpp +++ b/src/rtc/rtc_manager.cpp @@ -115,7 +115,12 @@ RTCManager::RTCManager( resolve(cf.vp8_decoder, info.vp8_decoders), resolve(cf.vp9_decoder, info.vp9_decoders), resolve(cf.av1_decoder, info.av1_decoders), - resolve(cf.h264_decoder, info.h264_decoders))); + resolve(cf.h264_decoder, info.h264_decoders) +#if USE_NVCODEC_ENCODER + , + cf.cuda_context +#endif + )); } media_dependencies.audio_mixer = nullptr; diff --git a/src/rtc/rtc_manager.h b/src/rtc/rtc_manager.h index 68bb13f5..23dd1b79 100644 --- a/src/rtc/rtc_manager.h +++ b/src/rtc/rtc_manager.h @@ -53,7 +53,7 @@ struct RTCManagerConfig { return webrtc::DegradationPreference::BALANCED; } -#if defined(__linux__) && USE_NVCODEC_ENCODER +#if USE_NVCODEC_ENCODER std::shared_ptr cuda_context; #endif }; diff --git a/src/video_codec_info.h b/src/video_codec_info.h index 9c8cf3a3..a71e4128 100644 --- a/src/video_codec_info.h +++ b/src/video_codec_info.h @@ -8,6 +8,7 @@ #if USE_NVCODEC_ENCODER #include "hwenc_nvcodec/nvcodec_h264_encoder.h" +#include "hwenc_nvcodec/nvcodec_video_decoder.h" #endif #if USE_JETSON_ENCODER @@ -104,6 +105,15 @@ struct VideoCodecInfo { if (NvCodecH264Encoder::IsSupported()) { info.h264_encoders.push_back(Type::NVIDIA); } + if (NvCodecVideoDecoder::IsSupported(CudaVideoCodec::VP8)) { + info.vp8_decoders.push_back(Type::NVIDIA); + } + if (NvCodecVideoDecoder::IsSupported(CudaVideoCodec::VP9)) { + info.vp9_decoders.push_back(Type::NVIDIA); + } + if (NvCodecVideoDecoder::IsSupported(CudaVideoCodec::H264)) { + info.h264_decoders.push_back(Type::NVIDIA); + } #endif info.vp8_encoders.push_back(Type::Software); @@ -142,6 +152,15 @@ struct VideoCodecInfo { if (NvCodecH264Encoder::IsSupported()) { info.h264_encoders.push_back(Type::NVIDIA); } + if (NvCodecVideoDecoder::IsSupported(CudaVideoCodec::VP8)) { + info.vp8_decoders.push_back(Type::NVIDIA); + } + if (NvCodecVideoDecoder::IsSupported(CudaVideoCodec::VP9)) { + info.vp9_decoders.push_back(Type::NVIDIA); + } + if (NvCodecVideoDecoder::IsSupported(CudaVideoCodec::H264)) { + info.h264_decoders.push_back(Type::NVIDIA); + } #endif #if USE_MMAL_ENCODER From ecca06ddeea5cc4f785fe5b865634048cf270ec8 Mon Sep 17 00:00:00 2001 From: melpon Date: Mon, 4 Apr 2022 02:16:37 +0900 Subject: [PATCH 26/70] =?UTF-8?q?Ubuntu=20=E3=81=AE=20CUDA=20=E3=82=92?= =?UTF-8?q?=E4=BD=BF=E3=81=A3=E3=81=9F=E3=83=8F=E3=83=BC=E3=83=89=E3=82=A6?= =?UTF-8?q?=E3=82=A7=E3=82=A2=E3=83=87=E3=82=B3=E3=83=BC=E3=83=80=E3=81=AB?= =?UTF-8?q?=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 5 +- script/get_cmake.sh | 1 + src/dyn/dyn.h | 250 ++++++++++---------- src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp | 4 +- 4 files changed, 131 insertions(+), 129 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 132eb3c7..fc5732ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -626,7 +626,8 @@ elseif (TARGET_OS STREQUAL "linux") src/hwenc_nvcodec/nvcodec_h264_encoder.cpp src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp - src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.cpp + src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp + src/hwenc_nvcodec/nvcodec_video_decoder.cpp NvCodec/NvCodec/NvDecoder/NvDecoder.cpp NvCodec/NvCodec/NvEncoder/NvEncoder.cpp NvCodec/NvCodec/NvEncoder/NvEncoderCuda.cpp) @@ -640,7 +641,7 @@ elseif (TARGET_OS STREQUAL "linux") set_source_files_properties( src/cuda/cuda_context_cuda.cpp src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp - src/hwenc_nvcodec/nvcodec_mjpeg_decoder_cuda.cpp + src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp NvCodec/NvCodec/NvDecoder/NvDecoder.cpp NvCodec/NvCodec/NvEncoder/NvEncoderCuda.cpp PROPERTIES diff --git a/script/get_cmake.sh b/script/get_cmake.sh index 5d6e8dda..869c230b 100755 --- a/script/get_cmake.sh +++ b/script/get_cmake.sh @@ -17,5 +17,6 @@ pushd $OUTPUT_DIR curl -LO https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-${PLATFORM}-x86_64.tar.gz tar xf cmake-${CMAKE_VERSION}-${PLATFORM}-x86_64.tar.gz rm cmake-${CMAKE_VERSION}-${PLATFORM}-x86_64.tar.gz + rm -rf cmake mv cmake-${CMAKE_VERSION}-${PLATFORM}-x86_64 cmake popd diff --git a/src/dyn/dyn.h b/src/dyn/dyn.h index 0b578c70..79ea7841 100644 --- a/src/dyn/dyn.h +++ b/src/dyn/dyn.h @@ -1,125 +1,125 @@ -#ifndef DYN_DYN_H_ -#define DYN_DYN_H_ - -#include -#include -#include - -#if defined(_WIN32) -#include -#elif defined(__linux__) -// Linux -#include -#endif - -namespace dyn { -class DynModule { - public: - static DynModule& Instance() { - static DynModule instance; - return instance; - } -#if defined(_WIN32) - typedef HINSTANCE__ module_t; - typedef HMODULE module_ptr_t; -#else - typedef void module_t; - typedef void* module_ptr_t; -#endif - - bool IsLoadable(const char* name) { -#if defined(_WIN32) - module_ptr_t module = LoadLibraryA(name); - if (module == nullptr) { - return false; - } - FreeLibrary(module); - return true; -#elif defined(__linux__) - module_ptr_t module = dlopen(name, RTLD_LAZY); - if (module == nullptr) { - return false; - } - dlclose(module); - return true; -#else - return false; -#endif - } - - module_ptr_t Get(const char* name) { - auto it = modules_.find(name); - if (it != modules_.end()) { - return it->second.get(); - } -#if defined(_WIN32) - module_ptr_t module = LoadLibraryA(name); -#elif defined(__linux__) - module_ptr_t module = dlopen(name, RTLD_LAZY); -#else - module_ptr_t module = nullptr; -#endif - if (module == nullptr) { - return nullptr; - } - modules_.insert(decltype(modules_)::value_type( - name, std::unique_ptr(module))); - return module; - } - - void* GetFunc(const char* soname, const char* name) { - module_ptr_t module = Get(soname); - if (module == nullptr) { - return nullptr; - } -#if defined(_WIN32) - return ::GetProcAddress(module, name); -#elif defined(__linux__) - return dlsym(module, name); -#else - return nullptr; -#endif - } - - private: - struct dlcloser { - void operator()(module_ptr_t p) { - if (p != nullptr) { -#if defined(_WIN32) - FreeLibrary(p); -#elif defined(__linux__) - ::dlclose(p); -#else -#endif - } - } - }; - - std::map> modules_; -}; -} // namespace dyn - -// text の定義を全て展開した上で文字列化する。 -// 単純に #text とした場合、全て展開する前に文字列化されてしまう -#define DYN_STRINGIZE(text) DYN_STRINGIZE_((text)) - -#if defined(_WIN32) -#define DYN_STRINGIZE_(x) DYN_STRINGIZE_I x -#else -#define DYN_STRINGIZE_(x) DYN_STRINGIZE_I ## x -#endif - -#define DYN_STRINGIZE_I(text) #text - -#define DYN_REGISTER(soname, func) \ - template \ - inline auto func(Args... args) { \ - typedef std::add_pointer::type func_type; \ - auto f = (func_type)DynModule::Instance().GetFunc(soname, DYN_STRINGIZE(func)); \ - if (f == nullptr) { \ - exit(1); \ - } \ - return f(args...); \ - } - -#endif // DYN_DYN_H_ \ No newline at end of file +#ifndef DYN_DYN_H_ +#define DYN_DYN_H_ + +#include +#include +#include + +#if defined(_WIN32) +#include +#elif defined(__linux__) +// Linux +#include +#endif + +namespace dyn { +class DynModule { + public: + static DynModule& Instance() { + static DynModule instance; + return instance; + } +#if defined(_WIN32) + typedef HINSTANCE__ module_t; + typedef HMODULE module_ptr_t; +#else + typedef void module_t; + typedef void* module_ptr_t; +#endif + + bool IsLoadable(const char* name) { +#if defined(_WIN32) + module_ptr_t module = LoadLibraryA(name); + if (module == nullptr) { + return false; + } + FreeLibrary(module); + return true; +#elif defined(__linux__) + module_ptr_t module = dlopen(name, RTLD_LAZY); + if (module == nullptr) { + return false; + } + dlclose(module); + return true; +#else + return false; +#endif + } + + module_ptr_t Get(const char* name) { + auto it = modules_.find(name); + if (it != modules_.end()) { + return it->second.get(); + } +#if defined(_WIN32) + module_ptr_t module = LoadLibraryA(name); +#elif defined(__linux__) + module_ptr_t module = dlopen(name, RTLD_LAZY); +#else + module_ptr_t module = nullptr; +#endif + if (module == nullptr) { + return nullptr; + } + modules_.insert(decltype(modules_)::value_type( + name, std::unique_ptr(module))); + return module; + } + + void* GetFunc(const char* soname, const char* name) { + module_ptr_t module = Get(soname); + if (module == nullptr) { + return nullptr; + } +#if defined(_WIN32) + return ::GetProcAddress(module, name); +#elif defined(__linux__) + return dlsym(module, name); +#else + return nullptr; +#endif + } + + private: + struct dlcloser { + void operator()(module_ptr_t p) { + if (p != nullptr) { +#if defined(_WIN32) + FreeLibrary(p); +#elif defined(__linux__) + ::dlclose(p); +#else +#endif + } + } + }; + + std::map> modules_; +}; +} // namespace dyn + +// text の定義を全て展開した上で文字列化する。 +// 単純に #text とした場合、全て展開する前に文字列化されてしまう +#if defined(_WIN32) +#define DYN_STRINGIZE(text) DYN_STRINGIZE_((text)) +#define DYN_STRINGIZE_(x) DYN_STRINGIZE_I x +#else +#define DYN_STRINGIZE(x) DYN_STRINGIZE_I(x) +#endif + +#define DYN_STRINGIZE_I(text) #text + +#define DYN_REGISTER(soname, func) \ + template \ + inline auto func(Args... args) { \ + typedef std::add_pointer::type func_type; \ + auto f = \ + (func_type)DynModule::Instance().GetFunc(soname, DYN_STRINGIZE(func)); \ + if (f == nullptr) { \ + exit(1); \ + } \ + return f(args...); \ + } + +#endif // DYN_DYN_H_ \ No newline at end of file diff --git a/src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp b/src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp index b84a197c..006ec0a0 100644 --- a/src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp +++ b/src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp @@ -52,8 +52,8 @@ rtc::scoped_refptr NvCodecV4L2Capturer::Create( rtc::scoped_refptr v4l2_capturer( new rtc::RefCountedObject()); - v4l2_capturer->decoder_.reset(new NvCodecDecoderCuda( - config.cuda_context, NvCodecDecoderCuda::VideoCodec::JPEG)); + v4l2_capturer->decoder_.reset( + new NvCodecDecoderCuda(config.cuda_context, CudaVideoCodec::JPEG)); if (v4l2_capturer->Init((const char*)&unique_name, config.video_device) < 0) { RTC_LOG(LS_WARNING) << "Failed to create NvCodecV4L2Capturer(" From 1ad613850b206d12b9551fc248b46157ef5b77d6 Mon Sep 17 00:00:00 2001 From: melpon Date: Mon, 4 Apr 2022 02:20:59 +0900 Subject: [PATCH 27/70] =?UTF-8?q?CHANGES=20=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index d1e2e9c7..fb528c80 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,8 @@ - [ADD] `--client-cert` と `--client-key` でクライアント認証をできるようにする - @melpon +- [ADD] Windows と Ubuntu で NVIDIA VIDEO CODEC SDK を使ったハードウェアデコーダに対応 + - @melpon - [UPDATE] CLI11 を 2.2.0 に上げる - @voluntas - [FIX] Ubuntu 20.04 + H.264 + サイマルキャスト + --hw-mjpeg-decoder true で落ちるのを修正 (#221) From aa650b1e4f2df6cd96def1eeae815fc048e712e7 Mon Sep 17 00:00:00 2001 From: voluntas Date: Mon, 4 Apr 2022 13:57:15 +0900 Subject: [PATCH 28/70] Update README.md --- README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 92b3901b..23513d89 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,11 @@ https://github.com/shiguredo/momo/releases ## 動作環境 + +- Raspberry Pi OS (64bit) ARMv8 + - Raspberry Pi 4 + - Raspberry Pi 3 + - Raspberry Pi 2 - Raspberry Pi OS (32bit) ARMv7 - Raspberry Pi 4 - Raspberry Pi 3 @@ -101,13 +106,6 @@ https://github.com/shiguredo/momo/releases - macOS 11 arm64 以降 - Windows 10.1809 x86_64 以降 -### 以下はビルドが通ること以外は確認していません - -- Raspberry Pi OS (64bit) ARMv8 - - Raspberry Pi 4 - - Raspberry Pi 3 - - Raspberry Pi 2 - ## 使ってみる Momo を使ってみたい人は [USE.md](doc/USE.md) をお読みください。 From cfae6bb2fd704d8b852239afde44b03d59edd5db Mon Sep 17 00:00:00 2001 From: voluntas Date: Mon, 4 Apr 2022 13:57:43 +0900 Subject: [PATCH 29/70] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 23513d89..380b7024 100644 --- a/README.md +++ b/README.md @@ -129,8 +129,8 @@ Momo を使ってみたい人は [USE.md](doc/USE.md) をお読みください Apache License 2.0 ``` -Copyright 2015-2021, tnoho (Original Author) -Copyright 2018-2021, Shiguredo Inc. +Copyright 2015-2022, tnoho (Original Author) +Copyright 2018-2022, Shiguredo Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From a56f9d50afabc5e7eb712149e4f7291ab82641e0 Mon Sep 17 00:00:00 2001 From: melpon Date: Mon, 4 Apr 2022 16:24:59 +0900 Subject: [PATCH 30/70] =?UTF-8?q?MMAL=20=E3=81=A7=20MJPEG=20=E3=83=87?= =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=89=E3=81=A7=E3=81=8D=E3=81=AA=E3=81=84?= =?UTF-8?q?=E3=81=93=E3=81=A8=E3=81=8C=E3=81=82=E3=82=8B=E3=81=AE=E3=81=A8?= =?UTF-8?q?=E3=80=81=E3=82=B5=E3=82=A4=E3=83=9E=E3=83=AB=E3=82=AD=E3=83=A3?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=81=A7=E5=8B=95=E3=81=8B=E3=81=AA=E3=81=84?= =?UTF-8?q?=E3=81=93=E3=81=A8=E3=81=8C=E3=81=82=E3=82=8B=E3=81=AE=E3=82=92?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_mmal/mmal_h264_encoder.cpp | 16 +++---- src/hwenc_mmal/mmal_v4l2_capturer.cpp | 61 ++++++++++++++++++++++----- src/hwenc_mmal/mmal_v4l2_capturer.h | 14 +++++- src/main.cpp | 2 + 4 files changed, 72 insertions(+), 21 deletions(-) diff --git a/src/hwenc_mmal/mmal_h264_encoder.cpp b/src/hwenc_mmal/mmal_h264_encoder.cpp index 9ef822c0..c1a43c36 100644 --- a/src/hwenc_mmal/mmal_h264_encoder.cpp +++ b/src/hwenc_mmal/mmal_h264_encoder.cpp @@ -458,17 +458,17 @@ int32_t MMALH264Encoder::Encode( offset += stride_width_; } offset = 0; - size_t offset_y = stride_width_ * stride_height_; - size_t width_uv = stride_width_ / 2; - size_t offset_v = (stride_height_ / 2) * width_uv; + size_t stride_uv = stride_width_ / 2; + size_t offset_u = stride_width_ * stride_height_; + size_t offset_v = offset_u + stride_uv * (stride_height_ / 2); for (size_t i = 0; i < ((i420_buffer->height() + 1) / 2); i++) { - memcpy(buffer->data + offset_y + offset, + memcpy(buffer->data + offset_u + offset, (uint8_t*)i420_buffer->DataU() + (i420_buffer->StrideU() * i), - width_uv); - memcpy(buffer->data + offset_y + offset_v + offset, + stride_uv); + memcpy(buffer->data + offset_v + offset, (uint8_t*)i420_buffer->DataV() + (i420_buffer->StrideV() * i), - width_uv); - offset += width_uv; + stride_uv); + offset += stride_uv; } buffer->length = buffer->alloc_size = webrtc::CalcBufferSize( webrtc::VideoType::kI420, stride_width_, stride_height_); diff --git a/src/hwenc_mmal/mmal_v4l2_capturer.cpp b/src/hwenc_mmal/mmal_v4l2_capturer.cpp index 840da7b1..015ed152 100644 --- a/src/hwenc_mmal/mmal_v4l2_capturer.cpp +++ b/src/hwenc_mmal/mmal_v4l2_capturer.cpp @@ -5,6 +5,7 @@ #include // WebRTC +#include #include #include #include @@ -47,7 +48,7 @@ rtc::scoped_refptr MMALV4L2Capturer::Create( return nullptr; } rtc::scoped_refptr v4l2_capturer( - new rtc::RefCountedObject()); + new rtc::RefCountedObject(config)); if (v4l2_capturer->Init((const char*)&unique_name, config.video_device) < 0) { RTC_LOG(LS_WARNING) << "Failed to create MMALV4L2Capturer(" << unique_name << ")"; @@ -62,13 +63,14 @@ rtc::scoped_refptr MMALV4L2Capturer::Create( return v4l2_capturer; } -MMALV4L2Capturer::MMALV4L2Capturer() +MMALV4L2Capturer::MMALV4L2Capturer(const MMALV4L2CapturerConfig& config) : component_in_(nullptr), decoder_(nullptr), resizer_(nullptr), connection_(nullptr), configured_width_(0), - configured_height_(0) { + configured_height_(0), + config_(config) { bcm_host_init(); decoded_buffer_num_ = 4; decoded_buffer_size_ = @@ -98,6 +100,30 @@ bool MMALV4L2Capturer::UseNativeBuffer() { } void MMALV4L2Capturer::OnCaptured(uint8_t* data, uint32_t bytesused) { + // やってくる MJPEG の種類によっては、MMAL が処理できないことがある。 + // その場合は APP0 マーカーを APP4 に書き換えると良いらしいので、 + // 適当に置き換えていく。 + // https://forums.raspberrypi.com/viewtopic.php?t=329233 + int n = 2; // SOI は飛ばす + while (n < bytesused - 1) { + // EOI または SOS マーカーが見つかったら終了する + if (data[n] == 0xff && data[n + 1] == 0xd9) { + break; + } + if (data[n] == 0xff && data[n + 1] == 0xda) { + break; + } + if (n >= bytesused - 3) { + break; + } + // APP0 マーカーだったら APP4 マーカーに置き換える + if (data[n] == 0xff && data[n + 1] == 0xe0) { + data[n + 1] = 0xe4; + } + int size = (int)data[n + 2] * 256 + (int)data[n + 3]; + n += 2 + size; + } + const int64_t timestamp_us = rtc::TimeMicros(); int adapted_width; @@ -213,14 +239,27 @@ void MMALV4L2Capturer::ResizerOutputCallback(MMAL_PORT_T* port, } } - rtc::scoped_refptr mmal_buffer( - MMALBuffer::Create(buffer, params->width, params->height)); - OnFrame(webrtc::VideoFrame::Builder() - .set_video_frame_buffer(mmal_buffer) - .set_timestamp_rtp(0) - .set_timestamp_us(buffer->pts) - .set_rotation(webrtc::kVideoRotation_0) - .build()); + if (config_.native_frame_output) { + rtc::scoped_refptr mmal_buffer( + MMALBuffer::Create(buffer, params->width, params->height)); + OnFrame(webrtc::VideoFrame::Builder() + .set_video_frame_buffer(mmal_buffer) + .set_timestamp_rtp(0) + .set_timestamp_us(buffer->pts) + .set_rotation(webrtc::kVideoRotation_0) + .build()); + } else { + auto i420_buffer = + webrtc::I420Buffer::Create(params->width, params->height); + std::memcpy(i420_buffer->MutableDataY(), buffer->data, buffer->length); + OnFrame(webrtc::VideoFrame::Builder() + .set_video_frame_buffer(i420_buffer) + .set_timestamp_rtp(0) + .set_timestamp_us(buffer->pts) + .set_rotation(webrtc::kVideoRotation_0) + .build()); + mmal_buffer_header_release(buffer); + } } int32_t MMALV4L2Capturer::MMALConfigure(int32_t width, int32_t height) { diff --git a/src/hwenc_mmal/mmal_v4l2_capturer.h b/src/hwenc_mmal/mmal_v4l2_capturer.h index 4065710d..2c80915d 100644 --- a/src/hwenc_mmal/mmal_v4l2_capturer.h +++ b/src/hwenc_mmal/mmal_v4l2_capturer.h @@ -24,13 +24,22 @@ extern "C" { #include "v4l2_video_capturer/v4l2_video_capturer.h" -typedef V4L2VideoCapturerConfig MMALV4L2CapturerConfig; +struct MMALV4L2CapturerConfig : V4L2VideoCapturerConfig { + MMALV4L2CapturerConfig(const V4L2VideoCapturerConfig& config) { + *static_cast(this) = config; + } + // native_frame_output == true の場合、MMAL のデータを kNative なフレームとして渡す。 + // native_frame_output == false の場合、データをコピーして I420Buffer なフレームを作って渡す。 + // 前者の方が効率が良いけれども、kNative なフレームはサイマルキャスト時に自動で + // リサイズしてくれないので、状況に応じて使い分けるのが良い。 + bool native_frame_output = false; +}; class MMALV4L2Capturer : public V4L2VideoCapturer { public: static rtc::scoped_refptr Create( MMALV4L2CapturerConfig config); - MMALV4L2Capturer(); + MMALV4L2Capturer(const MMALV4L2CapturerConfig& config); ~MMALV4L2Capturer(); private: @@ -74,6 +83,7 @@ class MMALV4L2Capturer : public V4L2VideoCapturer { std::queue> frame_params_; unsigned int decoded_buffer_num_; size_t decoded_buffer_size_; + MMALV4L2CapturerConfig config_; }; #endif // MMAL_V4L2_CAPTURER_H_ diff --git a/src/main.cpp b/src/main.cpp index c267718a..e9d70884 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -128,6 +128,8 @@ int main(int argc, char* argv[]) { #if USE_MMAL_ENCODER if (v4l2_config.use_native) { MMALV4L2CapturerConfig mmal_config = v4l2_config; + // サイマルキャストの場合はネイティブフレームを出力しない + mmal_config.native_frame_output = !(use_sora && args.sora_simulcast); return MMALV4L2Capturer::Create(std::move(mmal_config)); } else { return V4L2VideoCapturer::Create(std::move(v4l2_config)); From cb8dff20104be03569bc101079fdb649a4e120ca Mon Sep 17 00:00:00 2001 From: melpon Date: Mon, 4 Apr 2022 16:39:14 +0900 Subject: [PATCH 31/70] =?UTF-8?q?CHANGES=20=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index fb528c80..08ed4872 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,10 @@ - @voluntas - [FIX] Ubuntu 20.04 + H.264 + サイマルキャスト + --hw-mjpeg-decoder true で落ちるのを修正 (#221) - @melpon +- [FIX] Raspberry Pi + H.264 + --hw-mjpeg-decoder true で、カメラの種類によっては動かないことがあるのを修正 (#141) + - @melpon +- [FIX] Raspberry Pi + H.264 + サイマルキャスト + --hw-mjpeg-decoder true で動かないのを修正 (#236) + - @melpon ## 2022.1.0 From a2caa792e0403fecdaccb9e67e08c7d9017600ac Mon Sep 17 00:00:00 2001 From: melpon Date: Mon, 4 Apr 2022 18:47:28 +0900 Subject: [PATCH 32/70] =?UTF-8?q?=E5=80=A4=E3=82=92=E8=A8=AD=E5=AE=9A?= =?UTF-8?q?=E3=81=99=E3=82=8B=E5=89=8D=E3=81=AB=20=5FuseNative=20=E3=82=92?= =?UTF-8?q?=E5=88=A9=E7=94=A8=E3=81=97=E3=81=A6=E3=81=84=E3=81=9F=E3=81=AE?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/v4l2_video_capturer/v4l2_video_capturer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v4l2_video_capturer/v4l2_video_capturer.cpp b/src/v4l2_video_capturer/v4l2_video_capturer.cpp index 596cc813..d9bd3078 100644 --- a/src/v4l2_video_capturer/v4l2_video_capturer.cpp +++ b/src/v4l2_video_capturer/v4l2_video_capturer.cpp @@ -309,7 +309,7 @@ int32_t V4L2VideoCapturer::StartCapture(V4L2VideoCapturerConfig config) { // If driver doesn't support framerate control, need to hardcode. // Hardcoding the value based on the frame size. if (!driver_framerate_support) { - if (!_useNative && _currentWidth >= 800 && + if (!config.use_native && _currentWidth >= 800 && _captureVideoType != webrtc::VideoType::kMJPEG) { _currentFrameRate = 15; } else { From cab3af4d708a1aacb2ed0ea4dff6ef5b49977482 Mon Sep 17 00:00:00 2001 From: tnoho Date: Mon, 4 Apr 2022 23:11:48 +0900 Subject: [PATCH 33/70] =?UTF-8?q?SDL=20=E3=81=AE=20configure=20=E3=81=B8?= =?UTF-8?q?=E3=81=AE=E3=83=91=E3=83=83=E3=83=81=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/macos_arm64/install_deps.sh | 2 +- build/macos_x86_64/install_deps.sh | 2 +- patch/mac_sdl.patch | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/build/macos_arm64/install_deps.sh b/build/macos_arm64/install_deps.sh index 5cef54f4..be679bf6 100755 --- a/build/macos_arm64/install_deps.sh +++ b/build/macos_arm64/install_deps.sh @@ -117,7 +117,7 @@ if [ $SDL2_CHANGED -eq 1 -o ! -e $INSTALL_DIR/SDL2/lib/libSDL2.a ]; then mkdir -p $SOURCE_DIR/SDL2 mkdir -p $BUILD_DIR/SDL2 ../../script/setup_sdl2.sh $SDL2_VERSION $SOURCE_DIR/SDL2 - patch -uR $SOURCE_DIR/SDL2/source/configure < ../../patch/mac_sdl.patch + patch $SOURCE_DIR/SDL2/source/configure < ../../patch/mac_sdl.patch pushd $BUILD_DIR/SDL2 # SDL2 の CMakeLists.txt は Metal をサポートしてくれてないので、configure でビルドする # ref: https://bugzilla.libsdl.org/show_bug.cgi?id=4617 diff --git a/build/macos_x86_64/install_deps.sh b/build/macos_x86_64/install_deps.sh index 385ca87d..b22250f9 100755 --- a/build/macos_x86_64/install_deps.sh +++ b/build/macos_x86_64/install_deps.sh @@ -117,7 +117,7 @@ if [ $SDL2_CHANGED -eq 1 -o ! -e $INSTALL_DIR/SDL2/lib/libSDL2.a ]; then mkdir -p $SOURCE_DIR/SDL2 mkdir -p $BUILD_DIR/SDL2 ../../script/setup_sdl2.sh $SDL2_VERSION $SOURCE_DIR/SDL2 - patch -uR $SOURCE_DIR/SDL2/source/configure < ../../patch/mac_sdl.patch + patch $SOURCE_DIR/SDL2/source/configure < ../../patch/mac_sdl.patch pushd $BUILD_DIR/SDL2 # SDL2 の CMakeLists.txt は Metal をサポートしてくれてないので、configure でビルドする # ref: https://bugzilla.libsdl.org/show_bug.cgi?id=4617 diff --git a/patch/mac_sdl.patch b/patch/mac_sdl.patch index 4ff95562..a5dcede3 100644 --- a/patch/mac_sdl.patch +++ b/patch/mac_sdl.patch @@ -1,15 +1,15 @@ ---- configure 2022-03-17 22:44:00.181213036 +0900 -+++ configure 2022-03-17 22:42:21.470152564 +0900 +--- configure 2022-03-17 22:42:21.470152564 +0900 ++++ configure 2022-03-17 22:44:00.181213036 +0900 @@ -20459,9 +20459,9 @@ $as_echo "$have_gcc_declaration_after_statement" >&6; } CFLAGS="$save_CFLAGS" -- #if test x$have_gcc_declaration_after_statement = xyes; then -- # EXTRA_CFLAGS="$EXTRA_CFLAGS -Wdeclaration-after-statement -Werror=declaration-after-statement" -- #fi -+ if test x$have_gcc_declaration_after_statement = xyes; then -+ EXTRA_CFLAGS="$EXTRA_CFLAGS -Wdeclaration-after-statement -Werror=declaration-after-statement" -+ fi +- if test x$have_gcc_declaration_after_statement = xyes; then +- EXTRA_CFLAGS="$EXTRA_CFLAGS -Wdeclaration-after-statement -Werror=declaration-after-statement" +- fi ++ #if test x$have_gcc_declaration_after_statement = xyes; then ++ # EXTRA_CFLAGS="$EXTRA_CFLAGS -Wdeclaration-after-statement -Werror=declaration-after-statement" ++ #fi } CheckWarnAll() From 3228d7cecd64bbf81614dc0dc87f2885e53d075d Mon Sep 17 00:00:00 2001 From: tnoho Date: Mon, 4 Apr 2022 23:12:34 +0900 Subject: [PATCH 34/70] =?UTF-8?q?libwebrtc=20=E3=81=AE=E3=83=90=E3=83=BC?= =?UTF-8?q?=E3=82=B8=E3=83=A7=E3=83=B3=E3=82=92=20M100.4896@{#1}=20?= =?UTF-8?q?=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b7923247..d2437b2d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,7 +11,7 @@ ## develop -- [UPDATE] `libwebrtc` を `M89.4389@{#5}` に上げる +- [UPDATE] `libwebrtc` を `M100.4896@{#1}` に上げる - @tnoho - [FIX] libwebrtc m100 で make_ref_counted を使って scoped_refptr を作るようになったので修正 - @tnoho diff --git a/VERSION b/VERSION index c6d2a036..6b351a33 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ MOMO_VERSION=2022.1.0 -WEBRTC_BUILD_VERSION=100.4896.1.1 +WEBRTC_BUILD_VERSION=100.4896.1.2 BOOST_VERSION=1.78.0 CLI11_VERSION=2.1.2 SDL2_VERSION=2.0.20 From 7c4d17259fed26d1732c24adc94c49a1c9d71bf6 Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 5 Apr 2022 01:11:58 +0900 Subject: [PATCH 35/70] =?UTF-8?q?CUDA=20=E5=88=9D=E6=9C=9F=E5=8C=96?= =?UTF-8?q?=E5=A4=B1=E6=95=97=E6=99=82=E3=81=AB=E4=BE=8B=E5=A4=96=E3=82=92?= =?UTF-8?q?=E6=8A=95=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cuda/cuda_context_cuda.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/cuda/cuda_context_cuda.cpp b/src/cuda/cuda_context_cuda.cpp index 309cbdd0..75040eb9 100644 --- a/src/cuda/cuda_context_cuda.cpp +++ b/src/cuda/cuda_context_cuda.cpp @@ -18,16 +18,20 @@ struct CudaContextImpl { ~CudaContextImpl() { dyn::cuCtxDestroy(context); } }; +#define ckerror(call) \ + if (!ck(call)) \ + throw std::exception() + std::shared_ptr CudaContext::Create() { CUdevice device; CUcontext context; - ck(dyn::cuInit(0)); - ck(dyn::cuDeviceGet(&device, 0)); + ckerror(dyn::cuInit(0)); + ckerror(dyn::cuDeviceGet(&device, 0)); char device_name[80]; - ck(dyn::cuDeviceGetName(device_name, sizeof(device_name), device)); + ckerror(dyn::cuDeviceGetName(device_name, sizeof(device_name), device)); std::cout << "GPU in use: " << device_name << std::endl; - ck(dyn::cuCtxCreate(&context, 0, device)); + ckerror(dyn::cuCtxCreate(&context, 0, device)); std::shared_ptr impl(new CudaContextImpl()); impl->device = device; From 774ec889fa8f91f1286887e70af9d0940f57889f Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 6 Apr 2022 21:14:27 +0900 Subject: [PATCH 36/70] WIP --- .vscode/c_cpp_properties.json | 2 + CMakeLists.txt | 19 ++ VERSION | 4 +- cmake/FindLIBVA.cmake | 25 ++ cmake/FindMSDK.cmake | 20 ++ localbuild/build.ubuntu-20.04_x86_64.sh | 47 ++- src/hwenc_msdk/msdk_video_decoder.cpp | 262 ++++++++++++++++ src/hwenc_msdk/msdk_video_decoder.h | 57 ++++ src/hwenc_msdk/msdk_video_encoder.cpp | 388 ++++++++++++++++++++++++ src/hwenc_msdk/msdk_video_encoder.h | 68 +++++ src/hwenc_msdk/vaapi_utils.cpp | 53 ++++ src/hwenc_msdk/vaapi_utils.h | 52 ++++ src/hwenc_msdk/vaapi_utils_drm.cpp | 134 ++++++++ src/hwenc_msdk/vaapi_utils_drm.h | 43 +++ 14 files changed, 1172 insertions(+), 2 deletions(-) create mode 100644 cmake/FindLIBVA.cmake create mode 100644 cmake/FindMSDK.cmake create mode 100644 src/hwenc_msdk/msdk_video_decoder.cpp create mode 100644 src/hwenc_msdk/msdk_video_decoder.h create mode 100644 src/hwenc_msdk/msdk_video_encoder.cpp create mode 100644 src/hwenc_msdk/msdk_video_encoder.h create mode 100644 src/hwenc_msdk/vaapi_utils.cpp create mode 100644 src/hwenc_msdk/vaapi_utils.h create mode 100644 src/hwenc_msdk/vaapi_utils_drm.cpp create mode 100644 src/hwenc_msdk/vaapi_utils_drm.h diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index a8a1ac20..185ee35c 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -45,6 +45,8 @@ "${workspaceFolder}/_install/ubuntu-20.04_x86_64/boost/include", "${workspaceFolder}/_install/ubuntu-20.04_x86_64/CLI11/include", "${workspaceFolder}/_install/ubuntu-20.04_x86_64/SDL2/include/SDL2", + "${workspaceFolder}/_install/ubuntu-20.04_x86_64/msdk/include", + "${workspaceFolder}/_install/ubuntu-20.04_x86_64/libva/include", "${workspaceFolder}/_build/ubuntu-20.04_x86_64", "/usr/local/cuda/include" ], diff --git a/CMakeLists.txt b/CMakeLists.txt index fc5732ff..e024471f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,7 @@ set(TARGET_ARCH_ARM "" CACHE STRING "TARGET_ARCH が arm の場合の詳細な C set(USE_NVCODEC_ENCODER OFF CACHE BOOL "NVIDIA VIDEO CODEC SDK のハードウェアエンコーダを利用するかどうか") set(USE_MMAL_ENCODER OFF CACHE BOOL "MMAL ハードウェアエンコーダを利用するかどうか") set(USE_JETSON_ENCODER OFF CACHE BOOL "Jetson のハードウェアエンコーダを利用するかどうか") +set(USE_MSDK_ENCODER OFF CACHE BOOL "Intel Media SDK のハードウェアエンコーダを利用するかどうか") set(USE_H264 OFF CACHE BOOL "H264 を利用するかどうか") set(USE_SDL2 OFF CACHE BOOL "SDL2 による画面出力を利用するかどうか") set(USE_LINUX_PULSE_AUDIO OFF CACHE BOOL "Linux で ALSA の代わりに PulseAudio を利用するか") @@ -198,6 +199,7 @@ elseif(MOMO_PACKAGE_NAME STREQUAL "ubuntu-18.04_x86_64") set(USE_H264 ON) set(USE_SDL2 ON) set(USE_NVCODEC_ENCODER ON) + set(USE_MSDK_ENCODER ON) set(USE_SCREEN_CAPTURER ON) set(BOOST_ROOT_DIR /root/boost) set(CLI11_ROOT_DIR /root/CLI11) @@ -220,6 +222,7 @@ elseif(MOMO_PACKAGE_NAME STREQUAL "ubuntu-20.04_x86_64") set(USE_H264 ON) set(USE_SDL2 ON) set(USE_NVCODEC_ENCODER ON) + set(USE_MSDK_ENCODER ON) set(USE_SCREEN_CAPTURER ON) set(BOOST_ROOT_DIR /root/boost) set(CLI11_ROOT_DIR /root/CLI11) @@ -369,6 +372,7 @@ target_compile_definitions(momo USE_NVCODEC_ENCODER=$ USE_MMAL_ENCODER=$ USE_JETSON_ENCODER=$ + USE_MSDK_ENCODER=$ USE_H264=$ USE_SDL2=$ USE_LINUX_PULSE_AUDIO=$ @@ -652,6 +656,21 @@ elseif (TARGET_OS STREQUAL "linux") target_link_libraries(momo PRIVATE cudart_static dl rt) endif() + # Intel Media SDK + if (USE_MSDK_ENCODER) + find_package(MSDK REQUIRED) + target_sources(momo + PRIVATE + src/hwenc_msdk/msdk_video_decoder.cpp + src/hwenc_msdk/msdk_video_encoder.cpp + src/hwenc_msdk/vaapi_utils_drm.cpp + src/hwenc_msdk/vaapi_utils.cpp + ) + target_link_libraries(momo + PRIVATE + MSDK::MSDK) + endif() + if (TARGET_ARCH STREQUAL "arm") if (USE_LIBCXX) target_include_directories(momo PRIVATE ${SYSROOT}/usr/include/${ARCH_NAME}) diff --git a/VERSION b/VERSION index fc03fb15..6e274f60 100644 --- a/VERSION +++ b/VERSION @@ -5,4 +5,6 @@ CLI11_VERSION=2.2.0 SDL2_VERSION=2.0.20 CMAKE_VERSION=3.22.3 CUDA_VERSION=11.0.2-1 -WINCUDA_VERSION=10.2 \ No newline at end of file +WINCUDA_VERSION=10.2 +LIBVA_VERSION=2.14.0 +MSDK_VERSION=22.3.0 \ No newline at end of file diff --git a/cmake/FindLIBVA.cmake b/cmake/FindLIBVA.cmake new file mode 100644 index 00000000..d4d121ea --- /dev/null +++ b/cmake/FindLIBVA.cmake @@ -0,0 +1,25 @@ +find_path(LIBVA_INCLUDE_DIR NAMES va/va.h PATHS "${LIBVA_ROOT_DIR}/include") +find_library(LIBVA_LIBRARY NAMES va PATHS "${LIBVA_ROOT_DIR}/lib") +find_library(LIBVA_DRM_LIBRARY NAMES va-drm PATHS "${LIBVA_ROOT_DIR}/lib") + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(LIBVA DEFAULT_MSG LIBVA_LIBRARY LIBVA_DRM_LIBRARY LIBVA_INCLUDE_DIR) + +mark_as_advanced(LIBVA_INCLUDE_DIR LIBVA_LIBRARY LIBVA_DRM_LIBRARY) + +if(LIBVA_FOUND) + if(NOT TARGET LIBVA::LIBVA) + add_library(LIBVA::LIBVA UNKNOWN IMPORTED) + + set_target_properties(LIBVA::LIBVA PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${LIBVA_INCLUDE_DIR}" + IMPORTED_LOCATION "${LIBVA_LIBRARY}") + endif() + if(NOT TARGET LIBVA::LIBVA_DRM) + add_library(LIBVA::LIBVA_DRM UNKNOWN IMPORTED) + + set_target_properties(LIBVA::LIBVA_DRM PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${LIBVA_INCLUDE_DIR}" + IMPORTED_LOCATION "${LIBVA_DRM_LIBRARY}") + endif() +endif() \ No newline at end of file diff --git a/cmake/FindMSDK.cmake b/cmake/FindMSDK.cmake new file mode 100644 index 00000000..c1d630f7 --- /dev/null +++ b/cmake/FindMSDK.cmake @@ -0,0 +1,20 @@ +find_package(LIBVA REQUIRED) + +find_path(MSDK_INCLUDE_DIR NAMES mfx/mfxenc.h PATHS "${MSDK_ROOT_DIR}/include") +find_library(MSDK_LIBRARY NAMES mfx PATHS "${MSDK_ROOT_DIR}/lib") + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(MSDK DEFAULT_MSG MSDK_LIBRARY MSDK_INCLUDE_DIR) + +mark_as_advanced(MSDK_INCLUDE_DIR MSDK_LIBRARY) + +if(MSDK_FOUND) + if(NOT TARGET MSDK::MSDK) + add_library(MSDK::MSDK UNKNOWN IMPORTED) + + target_link_libraries(MSDK::MSDK INTERFACE LIBVA::LIBVA LIBVA::LIBVA_DRM) + set_target_properties(MSDK::MSDK PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${MSDK_INCLUDE_DIR}" + IMPORTED_LOCATION "${MSDK_LIBRARY}") + endif() +endif() \ No newline at end of file diff --git a/localbuild/build.ubuntu-20.04_x86_64.sh b/localbuild/build.ubuntu-20.04_x86_64.sh index 45c189ff..d04a8cde 100755 --- a/localbuild/build.ubuntu-20.04_x86_64.sh +++ b/localbuild/build.ubuntu-20.04_x86_64.sh @@ -4,7 +4,7 @@ cd `dirname $0` set -ex -# apt install cuda=$CUDA_VERSION clang-10 curl git libxtst-dev libxdamage-dev libxfixes-dev libxrandr-dev libxcomposite-dev +# apt install cuda=$CUDA_VERSION clang-10 curl git libxtst-dev libxdamage-dev libxfixes-dev libxrandr-dev libxcomposite-dev libtool PACKAGE_NAME=ubuntu-20.04_x86_64 PROJECT_DIR=`pwd`/.. @@ -34,6 +34,8 @@ if [ "$CURRENT_VERSION" != "$INSTALLED_VERSION" ]; then # Boost のビルド $SCRIPT_DIR/setup_boost.sh $BOOST_VERSION $SOURCE_DIR/boost $SOURCE_DIR pushd $SOURCE_DIR/boost/source + rm -rf $BUILD_DIR/boost + rm -rf $INSTALL_DIR/boost mkdir -p $BUILD_DIR/boost echo "using clang : : $INSTALL_DIR/llvm/clang/bin/clang++ : ;" > project-config.jam ./b2 \ @@ -126,6 +128,46 @@ if [ "$CURRENT_VERSION" != "$INSTALLED_VERSION" ]; then # apt-get update # DEBIAN_FRONTEND=noninteractive apt-get -y install cuda=$CUDA_VERSION clang-10 + # libva + rm -rf $SOURCE_DIR/libva + git clone --depth 1 --branch $LIBVA_VERSION https://github.com/intel/libva.git $SOURCE_DIR/libva + rm -rf $BUILD_DIR/libva + mkdir -p $BUILD_DIR/libva + pushd $BUILD_DIR/libva + CC=$INSTALL_DIR/llvm/clang/bin/clang \ + CXX=$INSTALL_DIR/llvm/clang/bin/clang++ \ + CFLAGS="-fPIC" \ + $SOURCE_DIR/libva/autogen.sh \ + --enable-static \ + --disable-shared \ + --prefix $INSTALL_DIR/libva + make -j`nproc` + rm -rf $INSTALL_DIR/libva + make install + popd + + # Intel Media SDK + rm -rf $SOURCE_DIR/msdk + git clone --depth 1 --branch intel-mediasdk-$MSDK_VERSION https://github.com/Intel-Media-SDK/MediaSDK.git $SOURCE_DIR/msdk + pushd $SOURCE_DIR/msdk + find . -name "CMakeLists.txt" | while read line; do sed -i 's/SHARED/STATIC/g' $line; done + popd + rm -rf $BUILD_DIR/msdk + mkdir -p $BUILD_DIR/msdk + pushd $BUILD_DIR/msdk + cmake \ + -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR/msdk \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH=$INSTALL_DIR/libva \ + -DCMAKE_C_COMPILER=$INSTALL_DIR/llvm/clang/bin/clang \ + -DCMAKE_CXX_COMPILER=$INSTALL_DIR/llvm/clang/bin/clang++ \ + -DBUILD_SAMPLES=OFF \ + -DBUILD_TUTORIALS=OFF \ + $SOURCE_DIR/msdk + cmake --build . -j`nproc` + cmake --install . + popd + cp $PROJECT_DIR/VERSION $INSTALL_DIR/VERSION fi @@ -153,12 +195,15 @@ pushd $BUILD_DIR/momo -DWEBRTC_INCLUDE_DIR=$INSTALL_DIR/webrtc/include \ -DWEBRTC_LIBRARY_DIR=$INSTALL_DIR/webrtc/lib \ -DLIBCXX_INCLUDE_DIR=$INSTALL_DIR/llvm/libcxx/include \ + -DLIBVA_ROOT_DIR=$INSTALL_DIR/libva \ + -DMSDK_ROOT_DIR=$INSTALL_DIR/msdk \ -DTARGET_OS="linux" \ -DTARGET_OS_LINUX="ubuntu-20.04" \ -DTARGET_ARCH="x86_64" \ -DUSE_H264=ON \ -DUSE_SDL2=ON \ -DUSE_NVCODEC_ENCODER=ON \ + -DUSE_MSDK_ENCODER=ON \ -DUSE_SCREEN_CAPTURER=ON \ -DCMAKE_C_COMPILER=clang-10 \ -DCMAKE_CXX_COMPILER=clang++-10 \ diff --git a/src/hwenc_msdk/msdk_video_decoder.cpp b/src/hwenc_msdk/msdk_video_decoder.cpp new file mode 100644 index 00000000..dc6851bd --- /dev/null +++ b/src/hwenc_msdk/msdk_video_decoder.cpp @@ -0,0 +1,262 @@ +#include "msdk_video_decoder.h" + +#include + +// Linux +#include + +// WebRTC +#include +#include +#include +#include +#include + +MsdkVideoDecoder::MsdkVideoDecoder(mfxU32 codec_id) + : codec_id_(codec_id), + decoder_(nullptr), + decode_complete_callback_(nullptr), + buffer_pool_(false, 300 /* max_number_of_buffers*/) { + mfxStatus sts = MFX_ERR_NONE; + + mfxIMPL impl = MFX_IMPL_HARDWARE; + mfxVersion ver = {{0, 1}}; + + // Initialize Intel Media SDK Session + sts = session_.Init(impl, &ver); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + // Create VA display + libva_ = CreateDRMLibVA(); + if (!libva_) { + RTC_LOG(LS_ERROR) << "Failed to create DRM VA display"; + throw 1; + } + + // Provide VA display handle to Media SDK + sts = session_.SetHandle(static_cast(MFX_HANDLE_VA_DISPLAY), + libva_->GetVADisplay()); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + // Query selected implementation and version + sts = session_.QueryIMPL(&impl); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + sts = session_.QueryVersion(&ver); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + RTC_LOG(LS_INFO) << "Intel Media SDK Implementation: " + << (impl == MFX_IMPL_SOFTWARE ? "SOFTWARE" : "HARDWARE"); + RTC_LOG(LS_INFO) << "Intel Media SDK API Version: " << ver.Major << "." + << ver.Minor; +} + +MsdkVideoDecoder::~MsdkVideoDecoder() { + Release(); +} + +bool MsdkVideoDecoder::Configure( + const webrtc::VideoDecoder::Settings& settings) { + width_ = settings.max_render_resolution().Width(); + height_ = settings.max_render_resolution().Height(); + + return InitMediaSDK(); +} + +int32_t MsdkVideoDecoder::Decode(const webrtc::EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) { + if (decoder_ == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (decode_complete_callback_ == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (input_image.data() == nullptr && input_image.size() > 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + if (bitstream_.MaxLength < bitstream_.DataLength + input_image.size()) { + bitstream_buffer_.resize(bitstream_.DataLength + input_image.size()); + bitstream_.MaxLength = bitstream_.DataLength + bitstream_buffer_.size(); + bitstream_.Data = bitstream_buffer_.data(); + } + //printf("size=%zu\n", input_image.size()); + //for (size_t i = 0; i < input_image.size(); i++) { + // const uint8_t* p = input_image.data(); + // if (i < 100) { + // printf(" %02x", p[i]); + // } else { + // printf("\n"); + // break; + // } + //} + + memmove(bitstream_.Data, bitstream_.Data + bitstream_.DataOffset, + bitstream_.DataLength); + bitstream_.DataOffset = 0; + memcpy(bitstream_.Data + bitstream_.DataLength, input_image.data(), + input_image.size()); + bitstream_.DataLength += input_image.size(); + + // 使ってない入力サーフェスを取り出す + auto surface = + std::find_if(surfaces_.begin(), surfaces_.end(), + [](const mfxFrameSurface1& s) { return !s.Data.Locked; }); + if (surface == surfaces_.end()) { + RTC_LOG(LS_ERROR) << "Surface not found"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // これだとキューイングしたデータとずれるので、本当は surface と一緒に保存して利用するべき + uint64_t pts = input_image.Timestamp(); + + mfxStatus sts; + mfxSyncPoint syncp; + mfxFrameSurface1* out_surface = nullptr; + //RTC_LOG(LS_ERROR) << "before DataOffset=" << bitstream_.DataOffset + // << " DataLength=" << bitstream_.DataLength; + while (true) { + sts = decoder_->DecodeFrameAsync(&bitstream_, &*surface, &out_surface, + &syncp); + if (sts == MFX_WRN_DEVICE_BUSY) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + break; + } + //RTC_LOG(LS_ERROR) << "after DataOffset=" << bitstream_.DataOffset + // << " DataLength=" << bitstream_.DataLength; + if (sts == MFX_ERR_MORE_DATA) { + // もっと入力が必要なので出直す + return WEBRTC_VIDEO_CODEC_OK; + } + if (!syncp) { + return WEBRTC_VIDEO_CODEC_OK; + } + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + sts = session_.SyncOperation(syncp, 600000); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + // NV12 から I420 に変換 + rtc::scoped_refptr i420_buffer = + buffer_pool_.CreateI420Buffer(width_, height_); + libyuv::NV12ToI420(out_surface->Data.Y, out_surface->Data.Pitch, + out_surface->Data.UV, out_surface->Data.Pitch, + i420_buffer->MutableDataY(), i420_buffer->StrideY(), + i420_buffer->MutableDataU(), i420_buffer->StrideU(), + i420_buffer->MutableDataV(), i420_buffer->StrideV(), + width_, height_); + + webrtc::VideoFrame decoded_image = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(i420_buffer) + .set_timestamp_rtp(pts) + .build(); + decode_complete_callback_->Decoded(decoded_image, absl::nullopt, + absl::nullopt); + + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t MsdkVideoDecoder::RegisterDecodeCompleteCallback( + webrtc::DecodedImageCallback* callback) { + decode_complete_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t MsdkVideoDecoder::Release() { + ReleaseMediaSDK(); + buffer_pool_.Release(); + return WEBRTC_VIDEO_CODEC_OK; +} + +const char* MsdkVideoDecoder::ImplementationName() const { + return "Intel Media SDK"; +} + +bool MsdkVideoDecoder::InitMediaSDK() { + decoder_.reset(new MFXVideoDECODE(session_)); + + mfxVideoParam param; + memset(¶m, 0, sizeof(param)); + + param.mfx.CodecId = codec_id_; + param.mfx.FrameInfo.FourCC = MFX_FOURCC_NV12; + param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV420; + param.mfx.FrameInfo.PicStruct = MFX_PICSTRUCT_PROGRESSIVE; + param.mfx.FrameInfo.CropX = 0; + param.mfx.FrameInfo.CropY = 0; + param.mfx.FrameInfo.CropW = width_; + param.mfx.FrameInfo.CropH = height_; + param.mfx.FrameInfo.Width = (width_ + 15) / 16 * 16; + param.mfx.FrameInfo.Height = (height_ + 15) / 16 * 16; + + param.mfx.GopRefDist = 1; + param.AsyncDepth = 1; + param.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY; + + //qmfxExtCodingOption ext_coding_option; + //qmemset(&ext_coding_option, 0, sizeof(ext_coding_option)); + //qext_coding_option.Header.BufferId = MFX_EXTBUFF_CODING_OPTION; + //qext_coding_option.Header.BufferSz = sizeof(ext_coding_option); + //qext_coding_option.MaxDecFrameBuffering = 1; + + //qmfxExtBuffer* ext_buffers[1]; + //qext_buffers[0] = (mfxExtBuffer*)&ext_coding_option; + //qparam.ExtParam = ext_buffers; + //qparam.NumExtParam = sizeof(ext_buffers) / sizeof(ext_buffers[0]); + + mfxStatus sts = MFX_ERR_NONE; + + sts = decoder_->Query(¶m, ¶m); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + // Query number of required surfaces for encoder + memset(&alloc_request_, 0, sizeof(alloc_request_)); + sts = decoder_->QueryIOSurf(¶m, &alloc_request_); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + RTC_LOG(LS_INFO) << "Decoder NumFrameSuggested=" + << alloc_request_.NumFrameSuggested; + + // Initialize the Media SDK encoder + sts = decoder_->Init(¶m); + //MSDK_IGNORE_MFX_STS(sts, MFX_WRN_PARTIAL_ACCELERATION); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + // 入力ビットストリーム + bitstream_buffer_.resize(1024 * 1024); + memset(&bitstream_, 0, sizeof(bitstream_)); + bitstream_.MaxLength = bitstream_buffer_.size(); + bitstream_.Data = bitstream_buffer_.data(); + + // 必要な枚数分の出力サーフェスを作る + { + int width = (alloc_request_.Info.Width + 31) / 32 * 32; + int height = (alloc_request_.Info.Height + 31) / 32 * 32; + // 1枚あたりのバイト数 + // NV12 なので 1 ピクセルあたり 12 ビット + int size = width * height * 12 / 8; + surface_buffer_.resize(alloc_request_.NumFrameSuggested * size); + + surfaces_.clear(); + surfaces_.reserve(alloc_request_.NumFrameSuggested); + for (int i = 0; i < alloc_request_.NumFrameSuggested; i++) { + mfxFrameSurface1 surface; + memset(&surface, 0, sizeof(surface)); + surface.Info = param.mfx.FrameInfo; + surface.Data.Y = surface_buffer_.data() + i * size; + surface.Data.U = surface_buffer_.data() + i * size + width * height; + surface.Data.V = surface_buffer_.data() + i * size + width * height + 1; + surface.Data.Pitch = width; + surfaces_.push_back(surface); + } + } + + return true; +} + +void MsdkVideoDecoder::ReleaseMediaSDK() { + decoder_.reset(); +} \ No newline at end of file diff --git a/src/hwenc_msdk/msdk_video_decoder.h b/src/hwenc_msdk/msdk_video_decoder.h new file mode 100644 index 00000000..7d868cfa --- /dev/null +++ b/src/hwenc_msdk/msdk_video_decoder.h @@ -0,0 +1,57 @@ +#ifndef MSDK_VIDEO_DECODER_H_ +#define MSDK_VIDEO_DECODER_H_ + +// WebRTC +#include +#include +#include + +// msdk +#include +#include +#include + +#include "vaapi_utils_drm.h" + +class MsdkVideoDecoder : public webrtc::VideoDecoder { + public: + // MFX_CODEC_AVC + // MFX_CODEC_VP8 + // MFX_CODEC_VP9 + MsdkVideoDecoder(mfxU32 codec_id); + ~MsdkVideoDecoder() override; + + bool Configure(const Settings& settings) override; + + int32_t Decode(const webrtc::EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) override; + + int32_t RegisterDecodeCompleteCallback( + webrtc::DecodedImageCallback* callback) override; + + int32_t Release() override; + + const char* ImplementationName() const override; + + private: + bool InitMediaSDK(); + void ReleaseMediaSDK(); + + int width_ = 0; + int height_ = 0; + webrtc::DecodedImageCallback* decode_complete_callback_ = nullptr; + webrtc::VideoFrameBufferPool buffer_pool_; + + mfxU32 codec_id_; + std::unique_ptr libva_; + MFXVideoSession session_; + mfxFrameAllocRequest alloc_request_; + std::unique_ptr decoder_; + std::vector surface_buffer_; + std::vector surfaces_; + std::vector bitstream_buffer_; + mfxBitstream bitstream_; +}; + +#endif // MSDK_VIDEO_DECODER_H_ \ No newline at end of file diff --git a/src/hwenc_msdk/msdk_video_encoder.cpp b/src/hwenc_msdk/msdk_video_encoder.cpp new file mode 100644 index 00000000..16dfbba3 --- /dev/null +++ b/src/hwenc_msdk/msdk_video_encoder.cpp @@ -0,0 +1,388 @@ +#include "msdk_video_encoder.h" + +// libyuv +#include + +const int kLowH264QpThreshold = 34; +const int kHighH264QpThreshold = 40; + +MsdkVideoEncoder::MsdkVideoEncoder(const cricket::VideoCodec& codec) + : bitrate_adjuster_(0.5, 0.95) { + mfxStatus sts = MFX_ERR_NONE; + + mfxIMPL impl = MFX_IMPL_HARDWARE; + mfxVersion ver = {{0, 1}}; + + // Initialize Intel Media SDK Session + sts = session_.Init(impl, &ver); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + // Create VA display + libva_ = CreateDRMLibVA(); + if (!libva_) { + RTC_LOG(LS_ERROR) << "Failed to create DRM VA display"; + throw 1; + } + + // Provide VA display handle to Media SDK + sts = session_.SetHandle(static_cast(MFX_HANDLE_VA_DISPLAY), + libva_->GetVADisplay()); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + // Query selected implementation and version + sts = session_.QueryIMPL(&impl); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + sts = session_.QueryVersion(&ver); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + RTC_LOG(LS_INFO) << "Intel Media SDK Implementation: " + << (impl == MFX_IMPL_SOFTWARE ? "SOFTWARE" : "HARDWARE"); + RTC_LOG(LS_INFO) << "Intel Media SDK API Version: " << ver.Major << "." + << ver.Minor; +} +MsdkVideoEncoder::~MsdkVideoEncoder() {} + +int32_t MsdkVideoEncoder::InitEncode(const webrtc::VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size) { + RTC_DCHECK(codec_settings); + RTC_DCHECK_EQ(codec_settings->codecType, webrtc::kVideoCodecH264); + + int32_t release_ret = Release(); + if (release_ret != WEBRTC_VIDEO_CODEC_OK) { + return release_ret; + } + + width_ = codec_settings->width; + height_ = codec_settings->height; + target_bitrate_bps_ = codec_settings->startBitrate * 1000; + max_bitrate_bps_ = codec_settings->maxBitrate * 1000; + bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); + framerate_ = codec_settings->maxFramerate; + mode_ = codec_settings->mode; + + RTC_LOG(LS_INFO) << "InitEncode " << target_bitrate_bps_ << "bit/sec"; + + // Initialize encoded image. Default buffer size: size of unencoded data. + encoded_image_._encodedWidth = 0; + encoded_image_._encodedHeight = 0; + encoded_image_.set_size(0); + encoded_image_.timing_.flags = + webrtc::VideoSendTiming::TimingFrameFlags::kInvalid; + encoded_image_.content_type_ = + (codec_settings->mode == webrtc::VideoCodecMode::kScreensharing) + ? webrtc::VideoContentType::SCREENSHARE + : webrtc::VideoContentType::UNSPECIFIED; + + return InitMediaSDK(); +} +int32_t MsdkVideoEncoder::RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback* callback) { + std::lock_guard lock(mutex_); + callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} +int32_t MsdkVideoEncoder::Release() { + return ReleaseMediaSDK(); +} +int32_t MsdkVideoEncoder::Encode( + const webrtc::VideoFrame& frame, + const std::vector* frame_types) { + bool send_key_frame = false; + + if (frame_types != nullptr) { + // We only support a single stream. + RTC_DCHECK_EQ(frame_types->size(), static_cast(1)); + // Skip frame? + if ((*frame_types)[0] == webrtc::VideoFrameType::kEmptyFrame) { + return WEBRTC_VIDEO_CODEC_OK; + } + // Force key frame? + send_key_frame = + (*frame_types)[0] == webrtc::VideoFrameType::kVideoFrameKey; + } + + // 使ってない入力サーフェスを取り出す + auto surface = + std::find_if(surfaces_.begin(), surfaces_.end(), + [](const mfxFrameSurface1& s) { return !s.Data.Locked; }); + if (surface == surfaces_.end()) { + RTC_LOG(LS_ERROR) << "Surface not found"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // I420 から NV12 に変換 + rtc::scoped_refptr frame_buffer = + frame.video_frame_buffer()->ToI420(); + libyuv::I420ToNV12( + frame_buffer->DataY(), frame_buffer->StrideY(), frame_buffer->DataU(), + frame_buffer->StrideU(), frame_buffer->DataV(), frame_buffer->StrideV(), + surface->Data.Y, surface->Data.Pitch, surface->Data.U, + surface->Data.Pitch, frame_buffer->width(), frame_buffer->height()); + + mfxStatus sts; + + mfxEncodeCtrl ctrl; + memset(&ctrl, 0, sizeof(ctrl)); + //send_key_frame = true; + if (send_key_frame) { + ctrl.FrameType = MFX_FRAMETYPE_I | MFX_FRAMETYPE_IDR | MFX_FRAMETYPE_REF; + } else { + ctrl.FrameType = MFX_FRAMETYPE_UNKNOWN; + } + + if (reconfigure_needed_) { + // 今の設定を取得する + mfxVideoParam param; + memset(¶m, 0, sizeof(param)); + + sts = encoder_->GetVideoParam(¶m); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + // ビットレートとフレームレートを変更する。 + // なお、encoder_->Reset() はキューイングしているサーフェスを + // 全て処理してから呼び出す必要がある。 + // ここでは encoder_->Init() の時に + // param.mfx.GopRefDist = 1; + // param.AsyncDepth = 1; + // ext_coding_option.MaxDecFrameBuffering = 1; + // を設定して、そもそもキューイングが起きないようにすることで対処している。 + param.mfx.TargetKbps = bitrate_adjuster_.GetAdjustedBitrateBps() / 1000; + param.mfx.FrameInfo.FrameRateExtN = framerate_; + param.mfx.FrameInfo.FrameRateExtD = 1; + + sts = encoder_->Reset(¶m); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + reconfigure_needed_ = false; + } + + // NV12 をハードウェアエンコード + mfxSyncPoint syncp; + sts = encoder_->EncodeFrameAsync(&ctrl, &*surface, &bitstream_, &syncp); + // alloc_request_.NumFrameSuggested が 1 の場合は MFX_ERR_MORE_DATA は発生しない + if (sts == MFX_ERR_MORE_DATA) { + // もっと入力が必要なので出直す + return WEBRTC_VIDEO_CODEC_OK; + } + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + sts = session_.SyncOperation(syncp, 600000); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + //RTC_LOG(LS_ERROR) << "SurfaceSize=" << (surface->Data.U - surface->Data.Y); + //RTC_LOG(LS_ERROR) << "DataLength=" << bitstream_.DataLength; + { + uint8_t* p = bitstream_.Data + bitstream_.DataOffset; + int size = bitstream_.DataLength; + bitstream_.DataLength = 0; + + //FILE* fp = fopen("test.mp4", "a+"); + //fwrite(p, 1, size, fp); + //fclose(fp); + + auto buf = webrtc::EncodedImageBuffer::Create(p, size); + encoded_image_.SetEncodedData(buf); + encoded_image_.set_size(size); + encoded_image_._encodedWidth = width_; + encoded_image_._encodedHeight = height_; + encoded_image_.content_type_ = + (mode_ == webrtc::VideoCodecMode::kScreensharing) + ? webrtc::VideoContentType::SCREENSHARE + : webrtc::VideoContentType::UNSPECIFIED; + encoded_image_.timing_.flags = webrtc::VideoSendTiming::kInvalid; + encoded_image_.SetTimestamp(frame.timestamp()); + encoded_image_.ntp_time_ms_ = frame.ntp_time_ms(); + encoded_image_.capture_time_ms_ = frame.render_time_ms(); + encoded_image_.rotation_ = frame.rotation(); + encoded_image_.SetColorSpace(frame.color_space()); + encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameDelta; + + webrtc::CodecSpecificInfo codec_specific; + codec_specific.codecType = webrtc::kVideoCodecH264; + codec_specific.codecSpecific.H264.packetization_mode = + webrtc::H264PacketizationMode::NonInterleaved; + + h264_bitstream_parser_.ParseBitstream(encoded_image_); + encoded_image_.qp_ = h264_bitstream_parser_.GetLastSliceQp().value_or(-1); + + webrtc::EncodedImageCallback::Result result = + callback_->OnEncodedImage(encoded_image_, &codec_specific); + if (result.error != webrtc::EncodedImageCallback::Result::OK) { + RTC_LOG(LS_ERROR) << __FUNCTION__ + << " OnEncodedImage failed error:" << result.error; + return WEBRTC_VIDEO_CODEC_ERROR; + } + bitrate_adjuster_.Update(size); + } + + return WEBRTC_VIDEO_CODEC_OK; +} +void MsdkVideoEncoder::SetRates(const RateControlParameters& parameters) { + if (parameters.framerate_fps < 1.0) { + RTC_LOG(LS_WARNING) << "Invalid frame rate: " << parameters.framerate_fps; + return; + } + + uint32_t new_framerate = (uint32_t)parameters.framerate_fps; + uint32_t new_bitrate = parameters.bitrate.get_sum_bps(); + RTC_LOG(LS_INFO) << __FUNCTION__ << " framerate_:" << framerate_ + << " new_framerate: " << new_framerate + << " target_bitrate_bps_:" << target_bitrate_bps_ + << " new_bitrate:" << new_bitrate + << " max_bitrate_bps_:" << max_bitrate_bps_; + framerate_ = new_framerate; + target_bitrate_bps_ = new_bitrate; + bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); + reconfigure_needed_ = true; +} +webrtc::VideoEncoder::EncoderInfo MsdkVideoEncoder::GetEncoderInfo() const { + webrtc::VideoEncoder::EncoderInfo info; + info.supports_native_handle = true; + info.implementation_name = "NvCodec H264"; + info.scaling_settings = webrtc::VideoEncoder::ScalingSettings( + kLowH264QpThreshold, kHighH264QpThreshold); + info.is_hardware_accelerated = true; + return info; +} + +int32_t MsdkVideoEncoder::InitMediaSDK() { + encoder_.reset(new MFXVideoENCODE(session_)); + + // Set required video parameters for encode + // - In this example we are encoding an AVC (H.264) stream + mfxVideoParam param; + memset(¶m, 0, sizeof(param)); + + param.mfx.CodecId = MFX_CODEC_AVC; + //param.mfx.CodecProfile = MFX_PROFILE_AVC_HIGH; + //param.mfx.CodecLevel = MFX_LEVEL_AVC_51; + param.mfx.CodecProfile = MFX_PROFILE_AVC_BASELINE; + param.mfx.CodecLevel = MFX_LEVEL_AVC_1; + param.mfx.TargetUsage = MFX_TARGETUSAGE_BALANCED; + param.mfx.TargetKbps = bitrate_adjuster_.GetAdjustedBitrateBps() / 1000; + param.mfx.MaxKbps = max_bitrate_bps_ / 1000; + param.mfx.RateControlMethod = MFX_RATECONTROL_VBR; + //param.mfx.NumSlice = 1; + //param.mfx.NumRefFrame = 1; + param.mfx.FrameInfo.FrameRateExtN = framerate_; + param.mfx.FrameInfo.FrameRateExtD = 1; + param.mfx.FrameInfo.FourCC = MFX_FOURCC_NV12; + param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV420; + param.mfx.FrameInfo.PicStruct = MFX_PICSTRUCT_PROGRESSIVE; + param.mfx.FrameInfo.CropX = 0; + param.mfx.FrameInfo.CropY = 0; + param.mfx.FrameInfo.CropW = width_; + param.mfx.FrameInfo.CropH = height_; + // Width must be a multiple of 16 + // Height must be a multiple of 16 in case of frame picture and a multiple of 32 in case of field picture + param.mfx.FrameInfo.Width = (width_ + 15) / 16 * 16; + param.mfx.FrameInfo.Height = (height_ + 15) / 16 * 16; + + //param.mfx.GopOptFlag = MFX_GOP_STRICT | MFX_GOP_CLOSED; + //param.mfx.IdrInterval = codec_settings->H264().keyFrameInterval; + //param.mfx.IdrInterval = 0; + param.mfx.GopRefDist = 1; + //param.mfx.EncodedOrder = 0; + param.AsyncDepth = 1; + param.IOPattern = + MFX_IOPATTERN_IN_SYSTEM_MEMORY | MFX_IOPATTERN_OUT_SYSTEM_MEMORY; + + mfxExtCodingOption ext_coding_option; + memset(&ext_coding_option, 0, sizeof(ext_coding_option)); + ext_coding_option.Header.BufferId = MFX_EXTBUFF_CODING_OPTION; + ext_coding_option.Header.BufferSz = sizeof(ext_coding_option); + ext_coding_option.AUDelimiter = MFX_CODINGOPTION_OFF; + ext_coding_option.MaxDecFrameBuffering = 1; + //ext_coding_option.NalHrdConformance = MFX_CODINGOPTION_OFF; + //ext_coding_option.VuiVclHrdParameters = MFX_CODINGOPTION_ON; + //ext_coding_option.SingleSeiNalUnit = MFX_CODINGOPTION_ON; + //ext_coding_option.RefPicMarkRep = MFX_CODINGOPTION_OFF; + //ext_coding_option.PicTimingSEI = MFX_CODINGOPTION_OFF; + //ext_coding_option.RecoveryPointSEI = MFX_CODINGOPTION_OFF; + //ext_coding_option.FramePicture = MFX_CODINGOPTION_OFF; + //ext_coding_option.FieldOutput = MFX_CODINGOPTION_ON; + + mfxExtCodingOption2 ext_coding_option2; + memset(&ext_coding_option2, 0, sizeof(ext_coding_option2)); + ext_coding_option2.Header.BufferId = MFX_EXTBUFF_CODING_OPTION2; + ext_coding_option2.Header.BufferSz = sizeof(ext_coding_option2); + ext_coding_option2.RepeatPPS = MFX_CODINGOPTION_ON; + //ext_coding_option2.MaxSliceSize = 1; + //ext_coding_option2.AdaptiveI = MFX_CODINGOPTION_ON; + + mfxExtBuffer* ext_buffers[2]; + ext_buffers[0] = (mfxExtBuffer*)&ext_coding_option; + ext_buffers[1] = (mfxExtBuffer*)&ext_coding_option2; + param.ExtParam = ext_buffers; + param.NumExtParam = sizeof(ext_buffers) / sizeof(ext_buffers[0]); + + mfxStatus sts = MFX_ERR_NONE; + + // Validate video encode parameters (optional) + // - In this example the validation result is written to same structure + // - MFX_WRN_INCOMPATIBLE_VIDEO_PARAM is returned if some of the video parameters are not supported, + // instead the encoder will select suitable parameters closest matching the requested configuration + sts = encoder_->Query(¶m, ¶m); + //MSDK_IGNORE_MFX_STS(sts, MFX_WRN_INCOMPATIBLE_VIDEO_PARAM); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + // Query number of required surfaces for encoder + memset(&alloc_request_, 0, sizeof(alloc_request_)); + sts = encoder_->QueryIOSurf(¶m, &alloc_request_); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + RTC_LOG(LS_INFO) << "Encoder NumFrameSuggested=" + << alloc_request_.NumFrameSuggested; + + // Initialize the Media SDK encoder + sts = encoder_->Init(¶m); + //MSDK_IGNORE_MFX_STS(sts, MFX_WRN_PARTIAL_ACCELERATION); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + // Retrieve video parameters selected by encoder. + // - BufferSizeInKB parameter is required to set bit stream buffer size + memset(¶m, 0, sizeof(param)); + sts = encoder_->GetVideoParam(¶m); + MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + RTC_LOG(LS_INFO) << "BufferSizeInKB=" << param.mfx.BufferSizeInKB; + + frame_info_ = param.mfx.FrameInfo; + + // 出力ビットストリームの初期化 + bitstream_buffer_.resize(param.mfx.BufferSizeInKB * 1000); + + memset(&bitstream_, 0, sizeof(bitstream_)); + bitstream_.MaxLength = bitstream_buffer_.size(); + bitstream_.Data = bitstream_buffer_.data(); + + // 必要な枚数分の入力サーフェスを作る + { + int width = (alloc_request_.Info.Width + 31) / 32 * 32; + int height = (alloc_request_.Info.Height + 31) / 32 * 32; + // 1枚あたりのバイト数 + // NV12 なので 1 ピクセルあたり 12 ビット + int size = width * height * 12 / 8; + surface_buffer_.resize(alloc_request_.NumFrameSuggested * size); + + surfaces_.clear(); + surfaces_.reserve(alloc_request_.NumFrameSuggested); + for (int i = 0; i < alloc_request_.NumFrameSuggested; i++) { + mfxFrameSurface1 surface; + memset(&surface, 0, sizeof(surface)); + surface.Info = frame_info_; + surface.Data.Y = surface_buffer_.data() + i * size; + surface.Data.U = surface_buffer_.data() + i * size + width * height; + surface.Data.V = surface_buffer_.data() + i * size + width * height + 1; + surface.Data.Pitch = width; + surfaces_.push_back(surface); + } + } + + return WEBRTC_VIDEO_CODEC_OK; +} +int32_t MsdkVideoEncoder::ReleaseMediaSDK() { + return WEBRTC_VIDEO_CODEC_OK; +} \ No newline at end of file diff --git a/src/hwenc_msdk/msdk_video_encoder.h b/src/hwenc_msdk/msdk_video_encoder.h new file mode 100644 index 00000000..b1e77d2b --- /dev/null +++ b/src/hwenc_msdk/msdk_video_encoder.h @@ -0,0 +1,68 @@ +#ifndef MSDK_VIDEO_ENCODER_H_ +#define MSDK_VIDEO_ENCODER_H_ + +#include +#include + +// WebRTC +#include +#include +#include +#include +#include +#include + +// msdk +#include + +#include "vaapi_utils_drm.h" + +class MsdkVideoEncoder : public webrtc::VideoEncoder { + public: + explicit MsdkVideoEncoder(const cricket::VideoCodec& codec); + ~MsdkVideoEncoder() override; + + int32_t InitEncode(const webrtc::VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size) override; + int32_t RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback* callback) override; + int32_t Release() override; + int32_t Encode( + const webrtc::VideoFrame& frame, + const std::vector* frame_types) override; + void SetRates(const RateControlParameters& parameters) override; + webrtc::VideoEncoder::EncoderInfo GetEncoderInfo() const override; + + private: + std::mutex mutex_; + webrtc::EncodedImageCallback* callback_ = nullptr; + webrtc::BitrateAdjuster bitrate_adjuster_; + uint32_t target_bitrate_bps_ = 0; + uint32_t max_bitrate_bps_ = 0; + bool reconfigure_needed_ = false; + bool use_native_ = false; + uint32_t width_ = 0; + uint32_t height_ = 0; + uint32_t framerate_ = 0; + webrtc::VideoCodecMode mode_ = webrtc::VideoCodecMode::kRealtimeVideo; + std::vector> v_packet_; + webrtc::EncodedImage encoded_image_; + webrtc::H264BitstreamParser h264_bitstream_parser_; + + int32_t InitMediaSDK(); + int32_t ReleaseMediaSDK(); + + std::vector surface_buffer_; + std::vector surfaces_; + + std::unique_ptr libva_; + MFXVideoSession session_; + mfxFrameAllocRequest alloc_request_; + std::unique_ptr encoder_; + std::vector bitstream_buffer_; + mfxBitstream bitstream_; + mfxFrameInfo frame_info_; +}; + +#endif // MSDK_VIDEO_ENCODER_H_ \ No newline at end of file diff --git a/src/hwenc_msdk/vaapi_utils.cpp b/src/hwenc_msdk/vaapi_utils.cpp new file mode 100644 index 00000000..eca5a637 --- /dev/null +++ b/src/hwenc_msdk/vaapi_utils.cpp @@ -0,0 +1,53 @@ +// https://github.com/Intel-Media-SDK/MediaSDK/blob/master/samples/sample_common/src/vaapi_utils.cpp より。 +// オリジナルのライセンスは以下。 +/******************************************************************************\ +Copyright (c) 2005-2019, Intel Corporation +All rights reserved. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +This sample was distributed or derived from the Intel's Media Samples package. +The original version of this sample may be obtained from https://software.intel.com/en-us/intel-media-server-studio +or https://software.intel.com/en-us/media-client-solutions-support. +\**********************************************************************************/ + +#include "vaapi_utils.h" + +mfxStatus va_to_mfx_status(VAStatus va_res) { + mfxStatus mfxRes = MFX_ERR_NONE; + + switch (va_res) { + case VA_STATUS_SUCCESS: + mfxRes = MFX_ERR_NONE; + break; + case VA_STATUS_ERROR_ALLOCATION_FAILED: + mfxRes = MFX_ERR_MEMORY_ALLOC; + break; + case VA_STATUS_ERROR_ATTR_NOT_SUPPORTED: + case VA_STATUS_ERROR_UNSUPPORTED_PROFILE: + case VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT: + case VA_STATUS_ERROR_UNSUPPORTED_RT_FORMAT: + case VA_STATUS_ERROR_UNSUPPORTED_BUFFERTYPE: + case VA_STATUS_ERROR_FLAG_NOT_SUPPORTED: + case VA_STATUS_ERROR_RESOLUTION_NOT_SUPPORTED: + mfxRes = MFX_ERR_UNSUPPORTED; + break; + case VA_STATUS_ERROR_INVALID_DISPLAY: + case VA_STATUS_ERROR_INVALID_CONFIG: + case VA_STATUS_ERROR_INVALID_CONTEXT: + case VA_STATUS_ERROR_INVALID_SURFACE: + case VA_STATUS_ERROR_INVALID_BUFFER: + case VA_STATUS_ERROR_INVALID_IMAGE: + case VA_STATUS_ERROR_INVALID_SUBPICTURE: + mfxRes = MFX_ERR_NOT_INITIALIZED; + break; + case VA_STATUS_ERROR_INVALID_PARAMETER: + mfxRes = MFX_ERR_INVALID_VIDEO_PARAM; + default: + mfxRes = MFX_ERR_UNKNOWN; + break; + } + return mfxRes; +} \ No newline at end of file diff --git a/src/hwenc_msdk/vaapi_utils.h b/src/hwenc_msdk/vaapi_utils.h new file mode 100644 index 00000000..522f0a86 --- /dev/null +++ b/src/hwenc_msdk/vaapi_utils.h @@ -0,0 +1,52 @@ +// https://github.com/Intel-Media-SDK/MediaSDK/blob/master/samples/sample_common/include/vaapi_utils.h より。 +// オリジナルのライセンスは以下。 +/******************************************************************************\ +Copyright (c) 2005-2019, Intel Corporation +All rights reserved. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +This sample was distributed or derived from the Intel's Media Samples package. +The original version of this sample may be obtained from https://software.intel.com/en-us/intel-media-server-studio +or https://software.intel.com/en-us/media-client-solutions-support. +\**********************************************************************************/ + +#ifndef VAAPI_UTILS_H_ +#define VAAPI_UTILS_H_ + +// msdk +#include + +// libva +#include + +class CLibVA { + public: + virtual ~CLibVA(void) {} + + VADisplay GetVADisplay() { return m_va_dpy; } + + protected: + CLibVA() : m_va_dpy(NULL) {} + VADisplay m_va_dpy; + + private: + CLibVA(CLibVA&&) = delete; + CLibVA(const CLibVA&) = delete; + CLibVA& operator=(CLibVA&&) = delete; + CLibVA& operator=(const CLibVA&) = delete; +}; + +mfxStatus va_to_mfx_status(VAStatus va_res); + +#define MSDK_CHECK_RESULT(P, X, ERR) \ + { \ + if ((X) > (P)) { \ + RTC_LOG(LS_ERROR) << "Intel Media SDK Error: " << ERR; \ + throw ERR; \ + } \ + } + +#endif // VAAPI_UTILS_H_ \ No newline at end of file diff --git a/src/hwenc_msdk/vaapi_utils_drm.cpp b/src/hwenc_msdk/vaapi_utils_drm.cpp new file mode 100644 index 00000000..016cc25f --- /dev/null +++ b/src/hwenc_msdk/vaapi_utils_drm.cpp @@ -0,0 +1,134 @@ +// https://github.com/Intel-Media-SDK/MediaSDK/blob/master/samples/sample_common/src/vaapi_utils_drm.cpp より。 +// オリジナルのライセンスは以下。 +/******************************************************************************\ +Copyright (c) 2005-2019, Intel Corporation +All rights reserved. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +This sample was distributed or derived from the Intel's Media Samples package. +The original version of this sample may be obtained from https://software.intel.com/en-us/intel-media-server-studio +or https://software.intel.com/en-us/media-client-solutions-support. +\**********************************************************************************/ +#include "vaapi_utils_drm.h" + +// Linux +#include +#include +#include +#include + +// WebRTC +#include + +// DRM +#include + +// libva +#include + +constexpr mfxU32 MFX_DRI_MAX_NODES_NUM = 16; +constexpr mfxU32 MFX_DRI_RENDER_START_INDEX = 128; +constexpr mfxU32 MFX_DRI_CARD_START_INDEX = 0; +constexpr mfxU32 MFX_DRM_DRIVER_NAME_LEN = 4; +const char* MFX_DRM_INTEL_DRIVER_NAME = "i915"; +const char* MFX_DRI_PATH = "/dev/dri/"; +const char* MFX_DRI_NODE_RENDER = "renderD"; +const char* MFX_DRI_NODE_CARD = "card"; + +int get_drm_driver_name(int fd, char* name, int name_size) { + drm_version_t version = {}; + version.name_len = name_size; + version.name = name; + return ioctl(fd, DRM_IOWR(0, drm_version), &version); +} + +int open_first_intel_adapter() { + std::string adapterPath = MFX_DRI_PATH; + char driverName[MFX_DRM_DRIVER_NAME_LEN + 1] = {}; + mfxU32 nodeIndex; + + adapterPath += MFX_DRI_NODE_RENDER; + nodeIndex = MFX_DRI_RENDER_START_INDEX; + + for (mfxU32 i = 0; i < MFX_DRI_MAX_NODES_NUM; ++i) { + std::string curAdapterPath = adapterPath + std::to_string(nodeIndex + i); + + int fd = open(curAdapterPath.c_str(), O_RDWR); + if (fd < 0) + continue; + + if (!get_drm_driver_name(fd, driverName, MFX_DRM_DRIVER_NAME_LEN) && + !strcmp(driverName, MFX_DRM_INTEL_DRIVER_NAME)) { + return fd; + } + close(fd); + } + + return -1; +} + +int open_intel_adapter(const std::string& devicePath) { + if (devicePath.empty()) + return open_first_intel_adapter(); + + int fd = open(devicePath.c_str(), O_RDWR); + + if (fd < 0) { + RTC_LOG(LS_ERROR) << "Failed to open specified device"; + return -1; + } + + char driverName[MFX_DRM_DRIVER_NAME_LEN + 1] = {}; + if (!get_drm_driver_name(fd, driverName, MFX_DRM_DRIVER_NAME_LEN) && + !strcmp(driverName, MFX_DRM_INTEL_DRIVER_NAME)) { + return fd; + } else { + close(fd); + RTC_LOG(LS_ERROR) << "Specified device is not Intel one"; + return -1; + } +} + +DRMLibVA::DRMLibVA(const std::string& devicePath) : CLibVA(), m_fd(-1) { + mfxStatus sts = MFX_ERR_NONE; + + m_fd = open_intel_adapter(devicePath); + if (m_fd < 0) + throw std::range_error("Intel GPU was not found"); + + m_va_dpy = vaGetDisplayDRM(m_fd); + if (m_va_dpy) { + int major_version = 0, minor_version = 0; + VAStatus va_res = vaInitialize(m_va_dpy, &major_version, &minor_version); + sts = va_to_mfx_status(va_res); + } else { + sts = MFX_ERR_NULL_PTR; + } + + if (MFX_ERR_NONE != sts) { + if (m_va_dpy) + vaTerminate(m_va_dpy); + close(m_fd); + throw std::runtime_error("Loading of VA display was failed"); + } +} + +DRMLibVA::~DRMLibVA(void) { + if (m_va_dpy) { + vaTerminate(m_va_dpy); + } + if (m_fd >= 0) { + close(m_fd); + } +} + +std::unique_ptr CreateDRMLibVA(const std::string& devicePath) { + try { + return std::unique_ptr(new DRMLibVA(devicePath)); + } catch (std::exception& e) { + return nullptr; + } +} \ No newline at end of file diff --git a/src/hwenc_msdk/vaapi_utils_drm.h b/src/hwenc_msdk/vaapi_utils_drm.h new file mode 100644 index 00000000..bd64d6a0 --- /dev/null +++ b/src/hwenc_msdk/vaapi_utils_drm.h @@ -0,0 +1,43 @@ +// https://github.com/Intel-Media-SDK/MediaSDK/blob/master/samples/sample_common/include/vaapi_utils_drm.h より。 +// オリジナルのライセンスは以下。 +/******************************************************************************\ +Copyright (c) 2005-2019, Intel Corporation +All rights reserved. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +This sample was distributed or derived from the Intel's Media Samples package. +The original version of this sample may be obtained from https://software.intel.com/en-us/intel-media-server-studio +or https://software.intel.com/en-us/media-client-solutions-support. +\**********************************************************************************/ + +#ifndef VAAPI_UTILS_DRM_H_ +#define VAAPI_UTILS_DRM_H_ + +#include +#include + +#include "vaapi_utils.h" + +class DRMLibVA : public CLibVA { + public: + DRMLibVA(const std::string& devicePath); + virtual ~DRMLibVA(void); + + inline int getFD() { return m_fd; } + + protected: + int m_fd; + + private: + DRMLibVA(DRMLibVA&&) = delete; + DRMLibVA(const DRMLibVA&) = delete; + DRMLibVA& operator=(DRMLibVA&&) = delete; + DRMLibVA& operator=(const DRMLibVA&) = delete; +}; + +std::unique_ptr CreateDRMLibVA(const std::string& devicePath = ""); + +#endif // VAAPI_UTILS_DRM_H_ \ No newline at end of file From 082f900a87e4d99ffc23fa6eb0ae05ad13c47d7b Mon Sep 17 00:00:00 2001 From: melpon Date: Thu, 7 Apr 2022 06:31:21 +0900 Subject: [PATCH 37/70] =?UTF-8?q?WSL2=20=E7=92=B0=E5=A2=83=E3=81=A7?= =?UTF-8?q?=E3=82=84=E3=82=8C=E3=82=8B=E3=81=A8=E3=81=93=E3=82=8D=E3=81=BE?= =?UTF-8?q?=E3=81=A7=E3=82=84=E3=81=A3=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/c_cpp_properties.json | 1 + NvCodec/Utils/NvCodecUtils.h | 8 +- cmake/FindLIBVA.cmake | 13 +++ src/hwenc_msdk/msdk_video_decoder.cpp | 71 ++++++++++++++ src/hwenc_msdk/msdk_video_decoder.h | 9 +- src/hwenc_msdk/msdk_video_encoder.cpp | 128 ++++++++++++++++++++++++++ src/hwenc_msdk/msdk_video_encoder.h | 7 ++ src/main.cpp | 6 +- src/video_codec_info.h | 32 +++++++ 9 files changed, 267 insertions(+), 8 deletions(-) diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 185ee35c..48afa950 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -53,6 +53,7 @@ "defines": [ "WEBRTC_LINUX", "USE_NVCODEC_ENCODER=1", + "USE_MSDK_ENCODER=1", "USE_MMAL_ENCODER=0", "USE_JETSON_ENCODER=0", "USE_H264=1", diff --git a/NvCodec/Utils/NvCodecUtils.h b/NvCodec/Utils/NvCodecUtils.h index ee623310..0b595fb6 100644 --- a/NvCodec/Utils/NvCodecUtils.h +++ b/NvCodec/Utils/NvCodecUtils.h @@ -38,7 +38,7 @@ inline bool check(CUresult e, int iLine, const char *szFile) { if (e != CUDA_SUCCESS) { const char *szErrName = NULL; dyn::cuGetErrorName(e, &szErrName); - LOG(FATAL) << "CUDA driver API error " << szErrName << " at line " << iLine << " in file " << szFile; + LOG(ERROR) << "CUDA driver API error " << szErrName << " at line " << iLine << " in file " << szFile; return false; } return true; @@ -48,7 +48,7 @@ inline bool check(CUresult e, int iLine, const char *szFile) { #ifdef __CUDA_RUNTIME_H__ inline bool check(cudaError_t e, int iLine, const char *szFile) { if (e != cudaSuccess) { - LOG(FATAL) << "CUDA runtime API error " << cudaGetErrorName(e) << " at line " << iLine << " in file " << szFile; + LOG(ERROR) << "CUDA runtime API error " << cudaGetErrorName(e) << " at line " << iLine << " in file " << szFile; return false; } return true; @@ -86,7 +86,7 @@ inline bool check(NVENCSTATUS e, int iLine, const char *szFile) { "NV_ENC_ERR_RESOURCE_NOT_MAPPED", }; if (e != NV_ENC_SUCCESS) { - LOG(FATAL) << "NVENC error " << aszErrName[e] << " at line " << iLine << " in file " << szFile; + LOG(ERROR) << "NVENC error " << aszErrName[e] << " at line " << iLine << " in file " << szFile; return false; } return true; @@ -98,7 +98,7 @@ inline bool check(HRESULT e, int iLine, const char *szFile) { if (e != S_OK) { std::stringstream stream; stream << std::hex << std::uppercase << e; - LOG(FATAL) << "HRESULT error 0x" << stream.str() << " at line " << iLine << " in file " << szFile; + LOG(ERROR) << "HRESULT error 0x" << stream.str() << " at line " << iLine << " in file " << szFile; return false; } return true; diff --git a/cmake/FindLIBVA.cmake b/cmake/FindLIBVA.cmake index d4d121ea..11b02893 100644 --- a/cmake/FindLIBVA.cmake +++ b/cmake/FindLIBVA.cmake @@ -1,3 +1,15 @@ +find_package(PkgConfig) +pkg_check_modules(LIBDRM libdrm) +if(LIBDRM_FOUND) + if(NOT TARGET LIBDRM::LIBDRM) + add_library(LIBDRM::LIBDRM UNKNOWN IMPORTED) + + set_target_properties(LIBDRM::LIBDRM PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${LIBDRM_INCLUDE_DIRS}" + IMPORTED_LOCATION "${LIBDRM_LINK_LIBRARIES}") + endif() +endif() + find_path(LIBVA_INCLUDE_DIR NAMES va/va.h PATHS "${LIBVA_ROOT_DIR}/include") find_library(LIBVA_LIBRARY NAMES va PATHS "${LIBVA_ROOT_DIR}/lib") find_library(LIBVA_DRM_LIBRARY NAMES va-drm PATHS "${LIBVA_ROOT_DIR}/lib") @@ -18,6 +30,7 @@ if(LIBVA_FOUND) if(NOT TARGET LIBVA::LIBVA_DRM) add_library(LIBVA::LIBVA_DRM UNKNOWN IMPORTED) + target_link_libraries(LIBVA::LIBVA_DRM INTERFACE LIBDRM::LIBDRM) set_target_properties(LIBVA::LIBVA_DRM PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LIBVA_INCLUDE_DIR}" IMPORTED_LOCATION "${LIBVA_DRM_LIBRARY}") diff --git a/src/hwenc_msdk/msdk_video_decoder.cpp b/src/hwenc_msdk/msdk_video_decoder.cpp index dc6851bd..63a76f37 100644 --- a/src/hwenc_msdk/msdk_video_decoder.cpp +++ b/src/hwenc_msdk/msdk_video_decoder.cpp @@ -55,6 +55,77 @@ MsdkVideoDecoder::~MsdkVideoDecoder() { Release(); } +bool MsdkVideoDecoder::IsSupported(mfxU32 codec) { + std::unique_ptr libva = CreateDRMLibVA(); + if (!libva) { + return false; + } + + MFXVideoSession session; + + mfxStatus sts = MFX_ERR_NONE; + + sts = session.SetHandle(static_cast(MFX_HANDLE_VA_DISPLAY), + libva->GetVADisplay()); + if (sts != MFX_ERR_NONE) { + return false; + } + + mfxIMPL impl = MFX_IMPL_HARDWARE; + sts = session.QueryIMPL(&impl); + if (sts != MFX_ERR_NONE) { + return false; + } + + mfxVersion ver = {{0, 1}}; + sts = session.QueryVersion(&ver); + if (sts != MFX_ERR_NONE) { + return false; + } + + int width = 640; + int height = 480; + + std::unique_ptr decoder; + decoder.reset(new MFXVideoDECODE(session)); + + mfxVideoParam param; + memset(¶m, 0, sizeof(param)); + + param.mfx.CodecId = codec; + param.mfx.FrameInfo.FourCC = MFX_FOURCC_NV12; + param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV420; + param.mfx.FrameInfo.PicStruct = MFX_PICSTRUCT_PROGRESSIVE; + param.mfx.FrameInfo.CropX = 0; + param.mfx.FrameInfo.CropY = 0; + param.mfx.FrameInfo.CropW = width; + param.mfx.FrameInfo.CropH = height; + param.mfx.FrameInfo.Width = (width + 15) / 16 * 16; + param.mfx.FrameInfo.Height = (height + 15) / 16 * 16; + + param.mfx.GopRefDist = 1; + param.AsyncDepth = 1; + param.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY; + + //qmfxExtCodingOption ext_coding_option; + //qmemset(&ext_coding_option, 0, sizeof(ext_coding_option)); + //qext_coding_option.Header.BufferId = MFX_EXTBUFF_CODING_OPTION; + //qext_coding_option.Header.BufferSz = sizeof(ext_coding_option); + //qext_coding_option.MaxDecFrameBuffering = 1; + + //qmfxExtBuffer* ext_buffers[1]; + //qext_buffers[0] = (mfxExtBuffer*)&ext_coding_option; + //qparam.ExtParam = ext_buffers; + //qparam.NumExtParam = sizeof(ext_buffers) / sizeof(ext_buffers[0]); + + sts = decoder->Query(¶m, ¶m); + if (sts == MFX_ERR_NONE || sts == MFX_ERR_UNSUPPORTED) { + return false; + } + + return true; +} + bool MsdkVideoDecoder::Configure( const webrtc::VideoDecoder::Settings& settings) { width_ = settings.max_render_resolution().Width(); diff --git a/src/hwenc_msdk/msdk_video_decoder.h b/src/hwenc_msdk/msdk_video_decoder.h index 7d868cfa..381019df 100644 --- a/src/hwenc_msdk/msdk_video_decoder.h +++ b/src/hwenc_msdk/msdk_video_decoder.h @@ -15,12 +15,15 @@ class MsdkVideoDecoder : public webrtc::VideoDecoder { public: - // MFX_CODEC_AVC - // MFX_CODEC_VP8 - // MFX_CODEC_VP9 MsdkVideoDecoder(mfxU32 codec_id); ~MsdkVideoDecoder() override; + // MFX_CODEC_VP8 + // MFX_CODEC_VP9 + // MFX_CODEC_AVC + // MFX_CODEC_AV1 + static bool IsSupported(mfxU32 codec); + bool Configure(const Settings& settings) override; int32_t Decode(const webrtc::EncodedImage& input_image, diff --git a/src/hwenc_msdk/msdk_video_encoder.cpp b/src/hwenc_msdk/msdk_video_encoder.cpp index 16dfbba3..0da8cfd0 100644 --- a/src/hwenc_msdk/msdk_video_encoder.cpp +++ b/src/hwenc_msdk/msdk_video_encoder.cpp @@ -43,6 +43,134 @@ MsdkVideoEncoder::MsdkVideoEncoder(const cricket::VideoCodec& codec) } MsdkVideoEncoder::~MsdkVideoEncoder() {} +bool MsdkVideoEncoder::IsSupported(mfxU32 codec) { + std::unique_ptr libva = CreateDRMLibVA(); + if (!libva) { + return false; + } + + MFXVideoSession session; + + mfxStatus sts = MFX_ERR_NONE; + + sts = session.SetHandle(static_cast(MFX_HANDLE_VA_DISPLAY), + libva->GetVADisplay()); + if (sts != MFX_ERR_NONE) { + return false; + } + + mfxIMPL impl = MFX_IMPL_HARDWARE; + sts = session.QueryIMPL(&impl); + if (sts != MFX_ERR_NONE) { + return false; + } + + mfxVersion ver = {{0, 1}}; + sts = session.QueryVersion(&ver); + if (sts != MFX_ERR_NONE) { + return false; + } + + std::unique_ptr encoder; + encoder.reset(new MFXVideoENCODE(session)); + + int framerate = 30; + int width = 640; + int height = 480; + + mfxVideoParam param; + memset(¶m, 0, sizeof(param)); + + param.mfx.CodecId = codec; + if (codec == MFX_CODEC_VP8) { + param.mfx.CodecProfile = MFX_PROFILE_VP8_0; + } else if (codec == MFX_CODEC_VP9) { + param.mfx.CodecProfile = MFX_PROFILE_VP9_0; + } else if (codec == MFX_CODEC_AVC) { + //param.mfx.CodecProfile = MFX_PROFILE_AVC_HIGH; + //param.mfx.CodecLevel = MFX_LEVEL_AVC_51; + param.mfx.CodecProfile = MFX_PROFILE_AVC_BASELINE; + param.mfx.CodecLevel = MFX_LEVEL_AVC_1; + } else if (codec == MFX_CODEC_AV1) { + param.mfx.CodecProfile = MFX_PROFILE_AV1_MAIN; + } + param.mfx.TargetUsage = MFX_TARGETUSAGE_BALANCED; + param.mfx.TargetKbps = 3000; + param.mfx.MaxKbps = 3000; + param.mfx.RateControlMethod = MFX_RATECONTROL_VBR; + //param.mfx.NumSlice = 1; + //param.mfx.NumRefFrame = 1; + param.mfx.FrameInfo.FrameRateExtN = framerate; + param.mfx.FrameInfo.FrameRateExtD = 1; + param.mfx.FrameInfo.FourCC = MFX_FOURCC_NV12; + param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV420; + param.mfx.FrameInfo.PicStruct = MFX_PICSTRUCT_PROGRESSIVE; + param.mfx.FrameInfo.CropX = 0; + param.mfx.FrameInfo.CropY = 0; + param.mfx.FrameInfo.CropW = width; + param.mfx.FrameInfo.CropH = height; + // Width must be a multiple of 16 + // Height must be a multiple of 16 in case of frame picture and a multiple of 32 in case of field picture + param.mfx.FrameInfo.Width = (width + 15) / 16 * 16; + param.mfx.FrameInfo.Height = (height + 15) / 16 * 16; + + //param.mfx.GopOptFlag = MFX_GOP_STRICT | MFX_GOP_CLOSED; + //param.mfx.IdrInterval = codec_settings->H264().keyFrameInterval; + //param.mfx.IdrInterval = 0; + param.mfx.GopRefDist = 1; + //param.mfx.EncodedOrder = 0; + param.AsyncDepth = 1; + param.IOPattern = + MFX_IOPATTERN_IN_SYSTEM_MEMORY | MFX_IOPATTERN_OUT_SYSTEM_MEMORY; + + mfxExtBuffer* ext_buffers[10]; + int ext_buffers_size = 0; + if (codec == MFX_CODEC_AVC) { + mfxExtCodingOption ext_coding_option; + memset(&ext_coding_option, 0, sizeof(ext_coding_option)); + ext_coding_option.Header.BufferId = MFX_EXTBUFF_CODING_OPTION; + ext_coding_option.Header.BufferSz = sizeof(ext_coding_option); + ext_coding_option.AUDelimiter = MFX_CODINGOPTION_OFF; + ext_coding_option.MaxDecFrameBuffering = 1; + //ext_coding_option.NalHrdConformance = MFX_CODINGOPTION_OFF; + //ext_coding_option.VuiVclHrdParameters = MFX_CODINGOPTION_ON; + //ext_coding_option.SingleSeiNalUnit = MFX_CODINGOPTION_ON; + //ext_coding_option.RefPicMarkRep = MFX_CODINGOPTION_OFF; + //ext_coding_option.PicTimingSEI = MFX_CODINGOPTION_OFF; + //ext_coding_option.RecoveryPointSEI = MFX_CODINGOPTION_OFF; + //ext_coding_option.FramePicture = MFX_CODINGOPTION_OFF; + //ext_coding_option.FieldOutput = MFX_CODINGOPTION_ON; + + mfxExtCodingOption2 ext_coding_option2; + memset(&ext_coding_option2, 0, sizeof(ext_coding_option2)); + ext_coding_option2.Header.BufferId = MFX_EXTBUFF_CODING_OPTION2; + ext_coding_option2.Header.BufferSz = sizeof(ext_coding_option2); + ext_coding_option2.RepeatPPS = MFX_CODINGOPTION_ON; + //ext_coding_option2.MaxSliceSize = 1; + //ext_coding_option2.AdaptiveI = MFX_CODINGOPTION_ON; + + ext_buffers[0] = (mfxExtBuffer*)&ext_coding_option; + ext_buffers[1] = (mfxExtBuffer*)&ext_coding_option2; + ext_buffers_size = 2; + } + + if (ext_buffers_size != 0) { + param.ExtParam = ext_buffers; + param.NumExtParam = ext_buffers_size; + } + + // MFX_ERR_NONE The function completed successfully. + // MFX_ERR_UNSUPPORTED The function failed to identify a specific implementation for the required features. + // MFX_WRN_PARTIAL_ACCELERATION The underlying hardware does not fully support the specified video parameters; The encoding may be partially accelerated. Only SDK HW implementations may return this status code. + // MFX_WRN_INCOMPATIBLE_VIDEO_PARAM The function detected some video parameters were incompatible with others; incompatibility resolved. + sts = encoder->Query(¶m, ¶m); + if (sts == MFX_ERR_NONE || sts == MFX_ERR_UNSUPPORTED) { + return false; + } + + return true; +} + int32_t MsdkVideoEncoder::InitEncode(const webrtc::VideoCodec* codec_settings, int32_t number_of_cores, size_t max_payload_size) { diff --git a/src/hwenc_msdk/msdk_video_encoder.h b/src/hwenc_msdk/msdk_video_encoder.h index b1e77d2b..7190400a 100644 --- a/src/hwenc_msdk/msdk_video_encoder.h +++ b/src/hwenc_msdk/msdk_video_encoder.h @@ -14,6 +14,7 @@ // msdk #include +#include #include "vaapi_utils_drm.h" @@ -22,6 +23,12 @@ class MsdkVideoEncoder : public webrtc::VideoEncoder { explicit MsdkVideoEncoder(const cricket::VideoCodec& codec); ~MsdkVideoEncoder() override; + // MFX_CODEC_VP8 + // MFX_CODEC_VP9 + // MFX_CODEC_AVC + // MFX_CODEC_AV1 + static bool IsSupported(mfxU32 codec); + int32_t InitEncode(const webrtc::VideoCodec* codec_settings, int32_t number_of_cores, size_t max_payload_size) override; diff --git a/src/main.cpp b/src/main.cpp index f52cc23b..b62dc4a5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -87,7 +87,11 @@ int main(int argc, char* argv[]) { rtc::LogMessage::AddLogToStream(log_sink.get(), rtc::LS_INFO); #if USE_NVCODEC_ENCODER - auto cuda_context = CudaContext::Create(); + std::shared_ptr cuda_context; + try { + cuda_context = CudaContext::Create(); + } catch (...) { + } #endif auto capturer = ([&]() -> rtc::scoped_refptr { diff --git a/src/video_codec_info.h b/src/video_codec_info.h index a71e4128..5a425729 100644 --- a/src/video_codec_info.h +++ b/src/video_codec_info.h @@ -11,6 +11,11 @@ #include "hwenc_nvcodec/nvcodec_video_decoder.h" #endif +#if USE_MSDK_ENCODER +#include "hwenc_msdk/msdk_video_decoder.h" +#include "hwenc_msdk/msdk_video_encoder.h" +#endif + #if USE_JETSON_ENCODER #include "hwenc_jetson/jetson_video_encoder.h" #endif @@ -163,6 +168,33 @@ struct VideoCodecInfo { } #endif +#if USE_MSDK_ENCODER + if (MsdkVideoEncoder::IsSupported(MFX_CODEC_VP8)) { + info.vp8_encoders.push_back(Type::Intel); + } + if (MsdkVideoEncoder::IsSupported(MFX_CODEC_VP9)) { + info.vp9_encoders.push_back(Type::Intel); + } + if (MsdkVideoEncoder::IsSupported(MFX_CODEC_AVC)) { + info.h264_encoders.push_back(Type::Intel); + } + if (MsdkVideoEncoder::IsSupported(MFX_CODEC_AV1)) { + info.av1_encoders.push_back(Type::Intel); + } + if (MsdkVideoDecoder::IsSupported(MFX_CODEC_VP8)) { + info.vp8_decoders.push_back(Type::Intel); + } + if (MsdkVideoDecoder::IsSupported(MFX_CODEC_VP9)) { + info.vp9_decoders.push_back(Type::Intel); + } + if (MsdkVideoDecoder::IsSupported(MFX_CODEC_AVC)) { + info.h264_decoders.push_back(Type::Intel); + } + if (MsdkVideoDecoder::IsSupported(MFX_CODEC_AV1)) { + info.av1_decoders.push_back(Type::Intel); + } +#endif + #if USE_MMAL_ENCODER info.h264_encoders.push_back(Type::MMAL); info.h264_decoders.push_back(Type::MMAL); From c468da56387eb221d77d09fe905b891078c34cee Mon Sep 17 00:00:00 2001 From: melpon Date: Fri, 8 Apr 2022 01:14:11 +0900 Subject: [PATCH 38/70] =?UTF-8?q?=E3=81=A8=E3=82=8A=E3=81=82=E3=81=88?= =?UTF-8?q?=E3=81=9A=E5=8B=95=E3=81=84=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 4 +- VERSION | 4 +- build/build.sh | 2 + build/ubuntu-20.04_x86_64/Dockerfile | 44 +++- cmake/FindLIBVA.cmake | 6 +- cmake/FindMSDK.cmake | 4 +- localbuild/build.ubuntu-20.04_x86_64.sh | 6 +- src/hwenc_msdk/msdk_session.cpp | 43 +++ src/hwenc_msdk/msdk_session.h | 23 ++ src/hwenc_msdk/msdk_video_decoder.cpp | 158 ++++------- src/hwenc_msdk/msdk_video_decoder.h | 17 +- src/hwenc_msdk/msdk_video_encoder.cpp | 334 ++++++++++++------------ src/hwenc_msdk/msdk_video_encoder.h | 20 +- src/main.cpp | 12 + src/rtc/momo_video_decoder_factory.cpp | 107 +++++--- src/rtc/momo_video_decoder_factory.h | 34 +-- src/rtc/momo_video_encoder_factory.cpp | 145 +++++----- src/rtc/momo_video_encoder_factory.h | 40 +-- src/rtc/rtc_manager.cpp | 45 ++-- src/rtc/rtc_manager.h | 7 + src/video_codec_info.h | 17 +- 21 files changed, 605 insertions(+), 467 deletions(-) create mode 100644 src/hwenc_msdk/msdk_session.cpp create mode 100644 src/hwenc_msdk/msdk_session.h diff --git a/CMakeLists.txt b/CMakeLists.txt index e024471f..a768b2d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -199,7 +199,6 @@ elseif(MOMO_PACKAGE_NAME STREQUAL "ubuntu-18.04_x86_64") set(USE_H264 ON) set(USE_SDL2 ON) set(USE_NVCODEC_ENCODER ON) - set(USE_MSDK_ENCODER ON) set(USE_SCREEN_CAPTURER ON) set(BOOST_ROOT_DIR /root/boost) set(CLI11_ROOT_DIR /root/CLI11) @@ -227,6 +226,8 @@ elseif(MOMO_PACKAGE_NAME STREQUAL "ubuntu-20.04_x86_64") set(BOOST_ROOT_DIR /root/boost) set(CLI11_ROOT_DIR /root/CLI11) set(SDL2_ROOT_DIR /root/SDL2) + set(LIBVA_ROOT_DIR /root/libva) + set(MSDK_ROOT_DIR /root/msdk) set(WEBRTC_INCLUDE_DIR /root/webrtc/include) set(WEBRTC_LIBRARY_DIR /root/webrtc/lib) @@ -661,6 +662,7 @@ elseif (TARGET_OS STREQUAL "linux") find_package(MSDK REQUIRED) target_sources(momo PRIVATE + src/hwenc_msdk/msdk_session.cpp src/hwenc_msdk/msdk_video_decoder.cpp src/hwenc_msdk/msdk_video_encoder.cpp src/hwenc_msdk/vaapi_utils_drm.cpp diff --git a/VERSION b/VERSION index 6e274f60..27c305f9 100644 --- a/VERSION +++ b/VERSION @@ -6,5 +6,5 @@ SDL2_VERSION=2.0.20 CMAKE_VERSION=3.22.3 CUDA_VERSION=11.0.2-1 WINCUDA_VERSION=10.2 -LIBVA_VERSION=2.14.0 -MSDK_VERSION=22.3.0 \ No newline at end of file +LIBVA_VERSION=2.7.0 +MSDK_VERSION=20.1.1 \ No newline at end of file diff --git a/build/build.sh b/build/build.sh index 05ea099e..b57eaec5 100755 --- a/build/build.sh +++ b/build/build.sh @@ -187,6 +187,8 @@ case "$PACKAGE" in --build-arg CLI11_VERSION=$CLI11_VERSION \ --build-arg CMAKE_VERSION=$CMAKE_VERSION \ --build-arg CUDA_VERSION=$CUDA_VERSION \ + --build-arg LIBVA_VERSION=$LIBVA_VERSION \ + --build-arg MSDK_VERSION=$MSDK_VERSION \ --build-arg PACKAGE_NAME=$PACKAGE \ $PACKAGE diff --git a/build/ubuntu-20.04_x86_64/Dockerfile b/build/ubuntu-20.04_x86_64/Dockerfile index cc59eb71..6a5e3689 100644 --- a/build/ubuntu-20.04_x86_64/Dockerfile +++ b/build/ubuntu-20.04_x86_64/Dockerfile @@ -11,7 +11,8 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloa COPY script/apt_install_x86_64.sh /root/ RUN --mount=type=cache,id=$PACKAGE_NAME,target=/var/cache/apt --mount=type=cache,id=$PACKAGE_NAME,target=/var/lib/apt \ - /root/apt_install_x86_64.sh + /root/apt_install_x86_64.sh \ + apt install libtool libdrm-dev # WebRTC の取得 @@ -135,3 +136,44 @@ RUN set -ex \ && add-apt-repository "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get -y install cuda=$CUDA_VERSION clang-10 + +# libva +ARG LIBVA_VERSION +RUN set -ex \ + && git clone --depth 1 --branch $LIBVA_VERSION https://github.com/intel/libva.git /root/libva-source \ + && mkdir -p /root/libva-build \ + && cd /root/libva-build \ + && CC=/root/llvm/clang/bin/clang \ + CXX=/root/llvm/clang/bin/clang++ \ + CFLAGS="-fPIC" \ + /root/libva-source/autogen.sh \ + --enable-static \ + --disable-shared \ + --with-drivers-path=/usr/lib/x86_64-linux-gnu/dri \ + --prefix /root/libva \ + && make -j`nproc` \ + && make install \ + && rm -rf /root/libva-build \ + && rm -rf /root/libva-source + +# Intel Media SDK +ARG MSDK_VERSION +RUN set -ex \ + && git clone --depth 1 --branch intel-mediasdk-$MSDK_VERSION https://github.com/Intel-Media-SDK/MediaSDK.git /root/msdk-source \ + && cd /root/msdk-source \ + && find . -name "CMakeLists.txt" | while read line; do sed -i 's/SHARED/STATIC/g' $line; done \ + && mkdir -p /root/msdk-build \ + && cd /root/msdk-build \ + && cmake \ + -DCMAKE_INSTALL_PREFIX=/root/msdk \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH=/root/libva \ + -DCMAKE_C_COMPILER=/root/llvm/clang/bin/clang \ + -DCMAKE_CXX_COMPILER=/root/llvm/clang/bin/clang++ \ + -DBUILD_SAMPLES=OFF \ + -DBUILD_TUTORIALS=OFF \ + /root/msdk-source \ + && cmake --build . -j`nproc` \ + && cmake --install . \ + && rm -rf /root/msdk-build \ + && rm -rf /root/msdk-source \ No newline at end of file diff --git a/cmake/FindLIBVA.cmake b/cmake/FindLIBVA.cmake index 11b02893..591a6551 100644 --- a/cmake/FindLIBVA.cmake +++ b/cmake/FindLIBVA.cmake @@ -10,9 +10,9 @@ if(LIBDRM_FOUND) endif() endif() -find_path(LIBVA_INCLUDE_DIR NAMES va/va.h PATHS "${LIBVA_ROOT_DIR}/include") -find_library(LIBVA_LIBRARY NAMES va PATHS "${LIBVA_ROOT_DIR}/lib") -find_library(LIBVA_DRM_LIBRARY NAMES va-drm PATHS "${LIBVA_ROOT_DIR}/lib") +find_path(LIBVA_INCLUDE_DIR NAMES va/va.h PATHS "${LIBVA_ROOT_DIR}/include" NO_DEFAULT_PATH) +find_library(LIBVA_LIBRARY NAMES va PATHS "${LIBVA_ROOT_DIR}/lib" NO_DEFAULT_PATH) +find_library(LIBVA_DRM_LIBRARY NAMES va-drm PATHS "${LIBVA_ROOT_DIR}/lib" NO_DEFAULT_PATH) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(LIBVA DEFAULT_MSG LIBVA_LIBRARY LIBVA_DRM_LIBRARY LIBVA_INCLUDE_DIR) diff --git a/cmake/FindMSDK.cmake b/cmake/FindMSDK.cmake index c1d630f7..43bcc1ce 100644 --- a/cmake/FindMSDK.cmake +++ b/cmake/FindMSDK.cmake @@ -1,7 +1,7 @@ find_package(LIBVA REQUIRED) -find_path(MSDK_INCLUDE_DIR NAMES mfx/mfxenc.h PATHS "${MSDK_ROOT_DIR}/include") -find_library(MSDK_LIBRARY NAMES mfx PATHS "${MSDK_ROOT_DIR}/lib") +find_path(MSDK_INCLUDE_DIR NAMES mfx/mfxenc.h PATHS "${MSDK_ROOT_DIR}/include" NO_DEFAULT_PATH) +find_library(MSDK_LIBRARY NAMES mfx PATHS "${MSDK_ROOT_DIR}/lib" NO_DEFAULT_PATH) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(MSDK DEFAULT_MSG MSDK_LIBRARY MSDK_INCLUDE_DIR) diff --git a/localbuild/build.ubuntu-20.04_x86_64.sh b/localbuild/build.ubuntu-20.04_x86_64.sh index d04a8cde..f56332d4 100755 --- a/localbuild/build.ubuntu-20.04_x86_64.sh +++ b/localbuild/build.ubuntu-20.04_x86_64.sh @@ -4,7 +4,10 @@ cd `dirname $0` set -ex -# apt install cuda=$CUDA_VERSION clang-10 curl git libxtst-dev libxdamage-dev libxfixes-dev libxrandr-dev libxcomposite-dev libtool +# apt install cuda=$CUDA_VERSION clang-10 curl git libxtst-dev libxdamage-dev libxfixes-dev libxrandr-dev libxcomposite-dev +# apt install libtool libdrm-dev +# 実行時には (libdrm-dev の代わりに) libdrm2 を入れる +# intel-media-va-driver または intel-media-va-driver-non-free を入れる PACKAGE_NAME=ubuntu-20.04_x86_64 PROJECT_DIR=`pwd`/.. @@ -140,6 +143,7 @@ if [ "$CURRENT_VERSION" != "$INSTALLED_VERSION" ]; then $SOURCE_DIR/libva/autogen.sh \ --enable-static \ --disable-shared \ + --with-drivers-path=/usr/lib/x86_64-linux-gnu/dri \ --prefix $INSTALL_DIR/libva make -j`nproc` rm -rf $INSTALL_DIR/libva diff --git a/src/hwenc_msdk/msdk_session.cpp b/src/hwenc_msdk/msdk_session.cpp new file mode 100644 index 00000000..dbdf9521 --- /dev/null +++ b/src/hwenc_msdk/msdk_session.cpp @@ -0,0 +1,43 @@ +#include "msdk_session.h" + +std::shared_ptr MsdkSession::Create() { + std::shared_ptr session(new MsdkSession()); + session->libva = CreateDRMLibVA(); + if (!session->libva) { + return nullptr; + } + + mfxStatus sts = MFX_ERR_NONE; + + mfxIMPL impl = MFX_IMPL_HARDWARE_ANY; + mfxVersion ver = {{0, 1}}; + + sts = session->session.Init(impl, &ver); + if (sts != MFX_ERR_NONE) { + return nullptr; + } + + sts = session->session.SetHandle( + static_cast(MFX_HANDLE_VA_DISPLAY), + session->libva->GetVADisplay()); + if (sts != MFX_ERR_NONE) { + return nullptr; + } + + // Query selected implementation and version + sts = session->session.QueryIMPL(&impl); + if (sts != MFX_ERR_NONE) { + return nullptr; + } + + sts = session->session.QueryVersion(&ver); + if (sts != MFX_ERR_NONE) { + return nullptr; + } + + //RTC_LOG(LS_INFO) << "Intel Media SDK Implementation: " + // << (impl == MFX_IMPL_SOFTWARE ? "SOFTWARE" : "HARDWARE"); + //RTC_LOG(LS_INFO) << "Intel Media SDK API Version: " << ver.Major << "." + // << ver.Minor; + return session; +} \ No newline at end of file diff --git a/src/hwenc_msdk/msdk_session.h b/src/hwenc_msdk/msdk_session.h new file mode 100644 index 00000000..5dfab113 --- /dev/null +++ b/src/hwenc_msdk/msdk_session.h @@ -0,0 +1,23 @@ +#ifndef MSDK_SESSION_H_ +#define MSDK_SESSION_H_ + +#include + +// Intel Media SDK +#include + +#ifdef __linux__ +#include "vaapi_utils_drm.h" +#endif + +struct MsdkSession { + MFXVideoSession session; + +#ifdef __linux__ + std::unique_ptr libva; +#endif + + static std::shared_ptr Create(); +}; + +#endif \ No newline at end of file diff --git a/src/hwenc_msdk/msdk_video_decoder.cpp b/src/hwenc_msdk/msdk_video_decoder.cpp index 63a76f37..a96b017f 100644 --- a/src/hwenc_msdk/msdk_video_decoder.cpp +++ b/src/hwenc_msdk/msdk_video_decoder.cpp @@ -1,5 +1,6 @@ #include "msdk_video_decoder.h" +#include #include // Linux @@ -12,83 +13,28 @@ #include #include -MsdkVideoDecoder::MsdkVideoDecoder(mfxU32 codec_id) - : codec_id_(codec_id), +MsdkVideoDecoder::MsdkVideoDecoder(std::shared_ptr session, + mfxU32 codec) + : session_(session), + codec_(codec), decoder_(nullptr), decode_complete_callback_(nullptr), - buffer_pool_(false, 300 /* max_number_of_buffers*/) { - mfxStatus sts = MFX_ERR_NONE; - - mfxIMPL impl = MFX_IMPL_HARDWARE; - mfxVersion ver = {{0, 1}}; - - // Initialize Intel Media SDK Session - sts = session_.Init(impl, &ver); - MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); - - // Create VA display - libva_ = CreateDRMLibVA(); - if (!libva_) { - RTC_LOG(LS_ERROR) << "Failed to create DRM VA display"; - throw 1; - } - - // Provide VA display handle to Media SDK - sts = session_.SetHandle(static_cast(MFX_HANDLE_VA_DISPLAY), - libva_->GetVADisplay()); - MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); - - // Query selected implementation and version - sts = session_.QueryIMPL(&impl); - MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); - - sts = session_.QueryVersion(&ver); - MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); - - RTC_LOG(LS_INFO) << "Intel Media SDK Implementation: " - << (impl == MFX_IMPL_SOFTWARE ? "SOFTWARE" : "HARDWARE"); - RTC_LOG(LS_INFO) << "Intel Media SDK API Version: " << ver.Major << "." - << ver.Minor; -} + buffer_pool_(false, 300 /* max_number_of_buffers*/) {} MsdkVideoDecoder::~MsdkVideoDecoder() { Release(); } -bool MsdkVideoDecoder::IsSupported(mfxU32 codec) { - std::unique_ptr libva = CreateDRMLibVA(); - if (!libva) { - return false; - } - - MFXVideoSession session; +std::unique_ptr MsdkVideoDecoder::CreateDecoder( + std::shared_ptr session, + mfxU32 codec, + int width, + int height, + bool init) { + std::unique_ptr decoder(new MFXVideoDECODE(session->session)); mfxStatus sts = MFX_ERR_NONE; - sts = session.SetHandle(static_cast(MFX_HANDLE_VA_DISPLAY), - libva->GetVADisplay()); - if (sts != MFX_ERR_NONE) { - return false; - } - - mfxIMPL impl = MFX_IMPL_HARDWARE; - sts = session.QueryIMPL(&impl); - if (sts != MFX_ERR_NONE) { - return false; - } - - mfxVersion ver = {{0, 1}}; - sts = session.QueryVersion(&ver); - if (sts != MFX_ERR_NONE) { - return false; - } - - int width = 640; - int height = 480; - - std::unique_ptr decoder; - decoder.reset(new MFXVideoDECODE(session)); - mfxVideoParam param; memset(¶m, 0, sizeof(param)); @@ -119,11 +65,34 @@ bool MsdkVideoDecoder::IsSupported(mfxU32 codec) { //qparam.NumExtParam = sizeof(ext_buffers) / sizeof(ext_buffers[0]); sts = decoder->Query(¶m, ¶m); - if (sts == MFX_ERR_NONE || sts == MFX_ERR_UNSUPPORTED) { - return false; + if (sts == MFX_ERR_UNSUPPORTED) { + return nullptr; } - return true; + if (sts != MFX_ERR_NONE) { + std::cerr << "Supported specified codec but has warning: sts=" << sts + << std::endl; + } + + if (init) { + // Initialize the Media SDK encoder + sts = decoder->Init(¶m); + if (sts != MFX_ERR_NONE) { + return nullptr; + } + } + + return decoder; +} + +bool MsdkVideoDecoder::IsSupported(std::shared_ptr session, + mfxU32 codec) { + int width = 640; + int height = 480; + + auto decoder = CreateDecoder(session, codec, 640, 480, false); + + return decoder != nullptr; } bool MsdkVideoDecoder::Configure( @@ -207,7 +176,7 @@ int32_t MsdkVideoDecoder::Decode(const webrtc::EncodedImage& input_image, } MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); - sts = session_.SyncOperation(syncp, 600000); + sts = session_->session.SyncOperation(syncp, 600000); MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); // NV12 から I420 に変換 @@ -247,41 +216,16 @@ const char* MsdkVideoDecoder::ImplementationName() const { } bool MsdkVideoDecoder::InitMediaSDK() { - decoder_.reset(new MFXVideoDECODE(session_)); - - mfxVideoParam param; - memset(¶m, 0, sizeof(param)); - - param.mfx.CodecId = codec_id_; - param.mfx.FrameInfo.FourCC = MFX_FOURCC_NV12; - param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV420; - param.mfx.FrameInfo.PicStruct = MFX_PICSTRUCT_PROGRESSIVE; - param.mfx.FrameInfo.CropX = 0; - param.mfx.FrameInfo.CropY = 0; - param.mfx.FrameInfo.CropW = width_; - param.mfx.FrameInfo.CropH = height_; - param.mfx.FrameInfo.Width = (width_ + 15) / 16 * 16; - param.mfx.FrameInfo.Height = (height_ + 15) / 16 * 16; - - param.mfx.GopRefDist = 1; - param.AsyncDepth = 1; - param.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY; - - //qmfxExtCodingOption ext_coding_option; - //qmemset(&ext_coding_option, 0, sizeof(ext_coding_option)); - //qext_coding_option.Header.BufferId = MFX_EXTBUFF_CODING_OPTION; - //qext_coding_option.Header.BufferSz = sizeof(ext_coding_option); - //qext_coding_option.MaxDecFrameBuffering = 1; - - //qmfxExtBuffer* ext_buffers[1]; - //qext_buffers[0] = (mfxExtBuffer*)&ext_coding_option; - //qparam.ExtParam = ext_buffers; - //qparam.NumExtParam = sizeof(ext_buffers) / sizeof(ext_buffers[0]); + decoder_ = CreateDecoder(session_, codec_, width_, height_, true); mfxStatus sts = MFX_ERR_NONE; - sts = decoder_->Query(¶m, ¶m); - MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + mfxVideoParam param; + memset(¶m, 0, sizeof(param)); + sts = decoder_->GetVideoParam(¶m); + if (sts != MFX_ERR_NONE) { + return false; + } // Query number of required surfaces for encoder memset(&alloc_request_, 0, sizeof(alloc_request_)); @@ -291,11 +235,6 @@ bool MsdkVideoDecoder::InitMediaSDK() { RTC_LOG(LS_INFO) << "Decoder NumFrameSuggested=" << alloc_request_.NumFrameSuggested; - // Initialize the Media SDK encoder - sts = decoder_->Init(¶m); - //MSDK_IGNORE_MFX_STS(sts, MFX_WRN_PARTIAL_ACCELERATION); - MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); - // 入力ビットストリーム bitstream_buffer_.resize(1024 * 1024); memset(&bitstream_, 0, sizeof(bitstream_)); @@ -329,5 +268,8 @@ bool MsdkVideoDecoder::InitMediaSDK() { } void MsdkVideoDecoder::ReleaseMediaSDK() { + if (decoder_ != nullptr) { + decoder_->Close(); + } decoder_.reset(); } \ No newline at end of file diff --git a/src/hwenc_msdk/msdk_video_decoder.h b/src/hwenc_msdk/msdk_video_decoder.h index 381019df..51187a90 100644 --- a/src/hwenc_msdk/msdk_video_decoder.h +++ b/src/hwenc_msdk/msdk_video_decoder.h @@ -11,18 +11,19 @@ #include #include +#include "msdk_session.h" #include "vaapi_utils_drm.h" class MsdkVideoDecoder : public webrtc::VideoDecoder { public: - MsdkVideoDecoder(mfxU32 codec_id); + MsdkVideoDecoder(std::shared_ptr session, mfxU32 codec); ~MsdkVideoDecoder() override; // MFX_CODEC_VP8 // MFX_CODEC_VP9 // MFX_CODEC_AVC // MFX_CODEC_AV1 - static bool IsSupported(mfxU32 codec); + static bool IsSupported(std::shared_ptr session, mfxU32 codec); bool Configure(const Settings& settings) override; @@ -38,6 +39,13 @@ class MsdkVideoDecoder : public webrtc::VideoDecoder { const char* ImplementationName() const override; private: + static std::unique_ptr CreateDecoder( + std::shared_ptr session, + mfxU32 codec, + int width, + int height, + bool init); + bool InitMediaSDK(); void ReleaseMediaSDK(); @@ -46,9 +54,8 @@ class MsdkVideoDecoder : public webrtc::VideoDecoder { webrtc::DecodedImageCallback* decode_complete_callback_ = nullptr; webrtc::VideoFrameBufferPool buffer_pool_; - mfxU32 codec_id_; - std::unique_ptr libva_; - MFXVideoSession session_; + mfxU32 codec_; + std::shared_ptr session_; mfxFrameAllocRequest alloc_request_; std::unique_ptr decoder_; std::vector surface_buffer_; diff --git a/src/hwenc_msdk/msdk_video_encoder.cpp b/src/hwenc_msdk/msdk_video_encoder.cpp index 0da8cfd0..6e272519 100644 --- a/src/hwenc_msdk/msdk_video_encoder.cpp +++ b/src/hwenc_msdk/msdk_video_encoder.cpp @@ -1,102 +1,50 @@ #include "msdk_video_encoder.h" +#include + // libyuv #include const int kLowH264QpThreshold = 34; const int kHighH264QpThreshold = 40; -MsdkVideoEncoder::MsdkVideoEncoder(const cricket::VideoCodec& codec) - : bitrate_adjuster_(0.5, 0.95) { - mfxStatus sts = MFX_ERR_NONE; - - mfxIMPL impl = MFX_IMPL_HARDWARE; - mfxVersion ver = {{0, 1}}; - - // Initialize Intel Media SDK Session - sts = session_.Init(impl, &ver); - MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); - - // Create VA display - libva_ = CreateDRMLibVA(); - if (!libva_) { - RTC_LOG(LS_ERROR) << "Failed to create DRM VA display"; - throw 1; - } - - // Provide VA display handle to Media SDK - sts = session_.SetHandle(static_cast(MFX_HANDLE_VA_DISPLAY), - libva_->GetVADisplay()); - MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); - - // Query selected implementation and version - sts = session_.QueryIMPL(&impl); - MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); - - sts = session_.QueryVersion(&ver); - MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); - - RTC_LOG(LS_INFO) << "Intel Media SDK Implementation: " - << (impl == MFX_IMPL_SOFTWARE ? "SOFTWARE" : "HARDWARE"); - RTC_LOG(LS_INFO) << "Intel Media SDK API Version: " << ver.Major << "." - << ver.Minor; -} +MsdkVideoEncoder::MsdkVideoEncoder(std::shared_ptr session, + mfxU32 codec) + : session_(session), codec_(codec), bitrate_adjuster_(0.5, 0.95) {} MsdkVideoEncoder::~MsdkVideoEncoder() {} -bool MsdkVideoEncoder::IsSupported(mfxU32 codec) { - std::unique_ptr libva = CreateDRMLibVA(); - if (!libva) { - return false; - } - - MFXVideoSession session; - +std::unique_ptr MsdkVideoEncoder::CreateEncoder( + std::shared_ptr session, + mfxU32 codec, + int width, + int height, + int framerate, + int target_kbps, + int max_kbps, + bool init) { mfxStatus sts = MFX_ERR_NONE; - sts = session.SetHandle(static_cast(MFX_HANDLE_VA_DISPLAY), - libva->GetVADisplay()); - if (sts != MFX_ERR_NONE) { - return false; - } - - mfxIMPL impl = MFX_IMPL_HARDWARE; - sts = session.QueryIMPL(&impl); - if (sts != MFX_ERR_NONE) { - return false; - } - - mfxVersion ver = {{0, 1}}; - sts = session.QueryVersion(&ver); - if (sts != MFX_ERR_NONE) { - return false; - } - - std::unique_ptr encoder; - encoder.reset(new MFXVideoENCODE(session)); - - int framerate = 30; - int width = 640; - int height = 480; - mfxVideoParam param; memset(¶m, 0, sizeof(param)); param.mfx.CodecId = codec; if (codec == MFX_CODEC_VP8) { - param.mfx.CodecProfile = MFX_PROFILE_VP8_0; + //param.mfx.CodecProfile = MFX_PROFILE_VP8_0; } else if (codec == MFX_CODEC_VP9) { - param.mfx.CodecProfile = MFX_PROFILE_VP9_0; + //param.mfx.CodecProfile = MFX_PROFILE_VP9_0; } else if (codec == MFX_CODEC_AVC) { //param.mfx.CodecProfile = MFX_PROFILE_AVC_HIGH; //param.mfx.CodecLevel = MFX_LEVEL_AVC_51; - param.mfx.CodecProfile = MFX_PROFILE_AVC_BASELINE; - param.mfx.CodecLevel = MFX_LEVEL_AVC_1; + //param.mfx.CodecProfile = MFX_PROFILE_AVC_MAIN; + //param.mfx.CodecLevel = MFX_LEVEL_AVC_1; } else if (codec == MFX_CODEC_AV1) { - param.mfx.CodecProfile = MFX_PROFILE_AV1_MAIN; + //param.mfx.CodecProfile = MFX_PROFILE_AV1_MAIN; } param.mfx.TargetUsage = MFX_TARGETUSAGE_BALANCED; - param.mfx.TargetKbps = 3000; - param.mfx.MaxKbps = 3000; + //param.mfx.BRCParamMultiplier = 1; + //param.mfx.InitialDelayInKB = target_kbps; + param.mfx.TargetKbps = target_kbps; + param.mfx.MaxKbps = max_kbps; param.mfx.RateControlMethod = MFX_RATECONTROL_VBR; //param.mfx.NumSlice = 1; //param.mfx.NumRefFrame = 1; @@ -124,9 +72,10 @@ bool MsdkVideoEncoder::IsSupported(mfxU32 codec) { MFX_IOPATTERN_IN_SYSTEM_MEMORY | MFX_IOPATTERN_OUT_SYSTEM_MEMORY; mfxExtBuffer* ext_buffers[10]; + mfxExtCodingOption ext_coding_option; + mfxExtCodingOption2 ext_coding_option2; int ext_buffers_size = 0; if (codec == MFX_CODEC_AVC) { - mfxExtCodingOption ext_coding_option; memset(&ext_coding_option, 0, sizeof(ext_coding_option)); ext_coding_option.Header.BufferId = MFX_EXTBUFF_CODING_OPTION; ext_coding_option.Header.BufferSz = sizeof(ext_coding_option); @@ -141,7 +90,6 @@ bool MsdkVideoEncoder::IsSupported(mfxU32 codec) { //ext_coding_option.FramePicture = MFX_CODINGOPTION_OFF; //ext_coding_option.FieldOutput = MFX_CODINGOPTION_ON; - mfxExtCodingOption2 ext_coding_option2; memset(&ext_coding_option2, 0, sizeof(ext_coding_option2)); ext_coding_option2.Header.BufferId = MFX_EXTBUFF_CODING_OPTION2; ext_coding_option2.Header.BufferSz = sizeof(ext_coding_option2); @@ -159,16 +107,103 @@ bool MsdkVideoEncoder::IsSupported(mfxU32 codec) { param.NumExtParam = ext_buffers_size; } + std::unique_ptr encoder(new MFXVideoENCODE(session->session)); + // MFX_ERR_NONE The function completed successfully. // MFX_ERR_UNSUPPORTED The function failed to identify a specific implementation for the required features. // MFX_WRN_PARTIAL_ACCELERATION The underlying hardware does not fully support the specified video parameters; The encoding may be partially accelerated. Only SDK HW implementations may return this status code. // MFX_WRN_INCOMPATIBLE_VIDEO_PARAM The function detected some video parameters were incompatible with others; incompatibility resolved. + mfxVideoParam bk_param; + memcpy(&bk_param, ¶m, sizeof(bk_param)); sts = encoder->Query(¶m, ¶m); - if (sts == MFX_ERR_NONE || sts == MFX_ERR_UNSUPPORTED) { - return false; + if (sts == MFX_ERR_UNSUPPORTED) { + memcpy(¶m, &bk_param, sizeof(bk_param)); + + // 失敗したら LowPower ON にした状態でもう一度確認する + param.mfx.LowPower = MFX_CODINGOPTION_ON; + if (codec == MFX_CODEC_AVC) { + param.mfx.RateControlMethod = MFX_RATECONTROL_CQP; + param.mfx.QPI = 25; + param.mfx.QPP = 33; + param.mfx.QPB = 40; + //param.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY; + } + memcpy(&bk_param, ¶m, sizeof(bk_param)); + sts = encoder->Query(¶m, ¶m); + if (sts == MFX_ERR_UNSUPPORTED) { + return nullptr; + } + } + + //#define F(NAME) \ + // if (bk_param.NAME != param.NAME) \ + // std::cout << "param " << #NAME << " old=" << bk_param.NAME \ + // << " new=" << param.NAME << std::endl + // + // F(mfx.LowPower); + // F(mfx.BRCParamMultiplier); + // F(mfx.FrameInfo.FrameRateExtN); + // F(mfx.FrameInfo.FrameRateExtD); + // F(mfx.FrameInfo.FourCC); + // F(mfx.FrameInfo.ChromaFormat); + // F(mfx.FrameInfo.PicStruct); + // F(mfx.FrameInfo.CropX); + // F(mfx.FrameInfo.CropY); + // F(mfx.FrameInfo.CropW); + // F(mfx.FrameInfo.CropH); + // F(mfx.FrameInfo.Width); + // F(mfx.FrameInfo.Height); + // F(mfx.CodecId); + // F(mfx.CodecProfile); + // F(mfx.CodecLevel); + // F(mfx.GopPicSize); + // F(mfx.GopRefDist); + // F(mfx.GopOptFlag); + // F(mfx.IdrInterval); + // F(mfx.TargetUsage); + // F(mfx.RateControlMethod); + // F(mfx.InitialDelayInKB); + // F(mfx.TargetKbps); + // F(mfx.MaxKbps); + // F(mfx.BufferSizeInKB); + // F(mfx.NumSlice); + // F(mfx.NumRefFrame); + // F(mfx.EncodedOrder); + // F(mfx.DecodedOrder); + // F(mfx.ExtendedPicStruct); + // F(mfx.TimeStampCalc); + // F(mfx.SliceGroupsPresent); + // F(mfx.MaxDecFrameBuffering); + // F(mfx.EnableReallocRequest); + // F(AsyncDepth); + // F(IOPattern); + //#undef F + + //if (sts != MFX_ERR_NONE) { + // const char* codec_str = codec == MFX_CODEC_VP8 ? "MFX_CODEC_VP8" + // : codec == MFX_CODEC_VP9 ? "MFX_CODEC_VP9" + // : codec == MFX_CODEC_AV1 ? "MFX_CODEC_AV1" + // : codec == MFX_CODEC_AVC ? "MFX_CODEC_AVC" + // : "MFX_CODEC_UNKNOWN"; + // std::cerr << "Supported specified codec but has warning: codec=" + // << codec_str << " sts=" << sts << std::endl; + //} + + if (init) { + sts = encoder->Init(¶m); + if (sts != MFX_ERR_NONE) { + RTC_LOG(LS_ERROR) << "Failed to Init: sts=" << sts; + return nullptr; + } } - return true; + return encoder; +} + +bool MsdkVideoEncoder::IsSupported(std::shared_ptr session, + mfxU32 codec) { + auto encoder = CreateEncoder(session, codec, 1920, 1080, 30, 10, 20, false); + return encoder != nullptr; } int32_t MsdkVideoEncoder::InitEncode(const webrtc::VideoCodec* codec_settings, @@ -261,6 +296,10 @@ int32_t MsdkVideoEncoder::Encode( } if (reconfigure_needed_) { + auto start_time = std::chrono::system_clock::now(); + RTC_LOG(LS_INFO) << "Start reconfigure: bps=" + << (bitrate_adjuster_.GetAdjustedBitrateBps() / 1000) + << " framerate=" << framerate_; // 今の設定を取得する mfxVideoParam param; memset(¶m, 0, sizeof(param)); @@ -276,7 +315,11 @@ int32_t MsdkVideoEncoder::Encode( // param.AsyncDepth = 1; // ext_coding_option.MaxDecFrameBuffering = 1; // を設定して、そもそもキューイングが起きないようにすることで対処している。 - param.mfx.TargetKbps = bitrate_adjuster_.GetAdjustedBitrateBps() / 1000; + if (param.mfx.RateControlMethod == MFX_RATECONTROL_CQP) { + //param.mfx.QPI = h264_bitstream_parser_.GetLastSliceQp().value_or(30); + } else { + param.mfx.TargetKbps = bitrate_adjuster_.GetAdjustedBitrateBps() / 1000; + } param.mfx.FrameInfo.FrameRateExtN = framerate_; param.mfx.FrameInfo.FrameRateExtD = 1; @@ -284,6 +327,13 @@ int32_t MsdkVideoEncoder::Encode( MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); reconfigure_needed_ = false; + + auto end_time = std::chrono::system_clock::now(); + RTC_LOG(LS_INFO) << "Finish reconfigure: " + << std::chrono::duration_cast( + end_time - start_time) + .count() + << " ms"; } // NV12 をハードウェアエンコード @@ -296,7 +346,7 @@ int32_t MsdkVideoEncoder::Encode( } MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); - sts = session_.SyncOperation(syncp, 600000); + sts = session_->session.SyncOperation(syncp, 600000); MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); //RTC_LOG(LS_ERROR) << "SurfaceSize=" << (surface->Data.U - surface->Data.Y); @@ -312,7 +362,6 @@ int32_t MsdkVideoEncoder::Encode( auto buf = webrtc::EncodedImageBuffer::Create(p, size); encoded_image_.SetEncodedData(buf); - encoded_image_.set_size(size); encoded_image_._encodedWidth = width_; encoded_image_._encodedHeight = height_; encoded_image_.content_type_ = @@ -325,12 +374,19 @@ int32_t MsdkVideoEncoder::Encode( encoded_image_.capture_time_ms_ = frame.render_time_ms(); encoded_image_.rotation_ = frame.rotation(); encoded_image_.SetColorSpace(frame.color_space()); - encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameDelta; + if (bitstream_.FrameType == MFX_FRAMETYPE_I || + bitstream_.FrameType == MFX_FRAMETYPE_IDR) { + encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameKey; + } else { + encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameDelta; + } webrtc::CodecSpecificInfo codec_specific; - codec_specific.codecType = webrtc::kVideoCodecH264; - codec_specific.codecSpecific.H264.packetization_mode = - webrtc::H264PacketizationMode::NonInterleaved; + if (codec_ == MFX_CODEC_AVC) { + codec_specific.codecType = webrtc::kVideoCodecH264; + codec_specific.codecSpecific.H264.packetization_mode = + webrtc::H264PacketizationMode::NonInterleaved; + } h264_bitstream_parser_.ParseBitstream(encoded_image_); encoded_image_.qp_ = h264_bitstream_parser_.GetLastSliceQp().value_or(-1); @@ -376,85 +432,24 @@ webrtc::VideoEncoder::EncoderInfo MsdkVideoEncoder::GetEncoderInfo() const { } int32_t MsdkVideoEncoder::InitMediaSDK() { - encoder_.reset(new MFXVideoENCODE(session_)); + encoder_ = CreateEncoder(session_, codec_, width_, height_, framerate_, + bitrate_adjuster_.GetAdjustedBitrateBps() / 1000, + max_bitrate_bps_ / 1000, true); + if (encoder_ == nullptr) { + RTC_LOG(LS_ERROR) << "Failed to create encoder"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + mfxStatus sts = MFX_ERR_NONE; - // Set required video parameters for encode - // - In this example we are encoding an AVC (H.264) stream mfxVideoParam param; memset(¶m, 0, sizeof(param)); - param.mfx.CodecId = MFX_CODEC_AVC; - //param.mfx.CodecProfile = MFX_PROFILE_AVC_HIGH; - //param.mfx.CodecLevel = MFX_LEVEL_AVC_51; - param.mfx.CodecProfile = MFX_PROFILE_AVC_BASELINE; - param.mfx.CodecLevel = MFX_LEVEL_AVC_1; - param.mfx.TargetUsage = MFX_TARGETUSAGE_BALANCED; - param.mfx.TargetKbps = bitrate_adjuster_.GetAdjustedBitrateBps() / 1000; - param.mfx.MaxKbps = max_bitrate_bps_ / 1000; - param.mfx.RateControlMethod = MFX_RATECONTROL_VBR; - //param.mfx.NumSlice = 1; - //param.mfx.NumRefFrame = 1; - param.mfx.FrameInfo.FrameRateExtN = framerate_; - param.mfx.FrameInfo.FrameRateExtD = 1; - param.mfx.FrameInfo.FourCC = MFX_FOURCC_NV12; - param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV420; - param.mfx.FrameInfo.PicStruct = MFX_PICSTRUCT_PROGRESSIVE; - param.mfx.FrameInfo.CropX = 0; - param.mfx.FrameInfo.CropY = 0; - param.mfx.FrameInfo.CropW = width_; - param.mfx.FrameInfo.CropH = height_; - // Width must be a multiple of 16 - // Height must be a multiple of 16 in case of frame picture and a multiple of 32 in case of field picture - param.mfx.FrameInfo.Width = (width_ + 15) / 16 * 16; - param.mfx.FrameInfo.Height = (height_ + 15) / 16 * 16; - - //param.mfx.GopOptFlag = MFX_GOP_STRICT | MFX_GOP_CLOSED; - //param.mfx.IdrInterval = codec_settings->H264().keyFrameInterval; - //param.mfx.IdrInterval = 0; - param.mfx.GopRefDist = 1; - //param.mfx.EncodedOrder = 0; - param.AsyncDepth = 1; - param.IOPattern = - MFX_IOPATTERN_IN_SYSTEM_MEMORY | MFX_IOPATTERN_OUT_SYSTEM_MEMORY; - - mfxExtCodingOption ext_coding_option; - memset(&ext_coding_option, 0, sizeof(ext_coding_option)); - ext_coding_option.Header.BufferId = MFX_EXTBUFF_CODING_OPTION; - ext_coding_option.Header.BufferSz = sizeof(ext_coding_option); - ext_coding_option.AUDelimiter = MFX_CODINGOPTION_OFF; - ext_coding_option.MaxDecFrameBuffering = 1; - //ext_coding_option.NalHrdConformance = MFX_CODINGOPTION_OFF; - //ext_coding_option.VuiVclHrdParameters = MFX_CODINGOPTION_ON; - //ext_coding_option.SingleSeiNalUnit = MFX_CODINGOPTION_ON; - //ext_coding_option.RefPicMarkRep = MFX_CODINGOPTION_OFF; - //ext_coding_option.PicTimingSEI = MFX_CODINGOPTION_OFF; - //ext_coding_option.RecoveryPointSEI = MFX_CODINGOPTION_OFF; - //ext_coding_option.FramePicture = MFX_CODINGOPTION_OFF; - //ext_coding_option.FieldOutput = MFX_CODINGOPTION_ON; - - mfxExtCodingOption2 ext_coding_option2; - memset(&ext_coding_option2, 0, sizeof(ext_coding_option2)); - ext_coding_option2.Header.BufferId = MFX_EXTBUFF_CODING_OPTION2; - ext_coding_option2.Header.BufferSz = sizeof(ext_coding_option2); - ext_coding_option2.RepeatPPS = MFX_CODINGOPTION_ON; - //ext_coding_option2.MaxSliceSize = 1; - //ext_coding_option2.AdaptiveI = MFX_CODINGOPTION_ON; - - mfxExtBuffer* ext_buffers[2]; - ext_buffers[0] = (mfxExtBuffer*)&ext_coding_option; - ext_buffers[1] = (mfxExtBuffer*)&ext_coding_option2; - param.ExtParam = ext_buffers; - param.NumExtParam = sizeof(ext_buffers) / sizeof(ext_buffers[0]); - - mfxStatus sts = MFX_ERR_NONE; - - // Validate video encode parameters (optional) - // - In this example the validation result is written to same structure - // - MFX_WRN_INCOMPATIBLE_VIDEO_PARAM is returned if some of the video parameters are not supported, - // instead the encoder will select suitable parameters closest matching the requested configuration - sts = encoder_->Query(¶m, ¶m); - //MSDK_IGNORE_MFX_STS(sts, MFX_WRN_INCOMPATIBLE_VIDEO_PARAM); + // Retrieve video parameters selected by encoder. + // - BufferSizeInKB parameter is required to set bit stream buffer size + sts = encoder_->GetVideoParam(¶m); MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + RTC_LOG(LS_INFO) << "BufferSizeInKB=" << param.mfx.BufferSizeInKB; // Query number of required surfaces for encoder memset(&alloc_request_, 0, sizeof(alloc_request_)); @@ -464,19 +459,6 @@ int32_t MsdkVideoEncoder::InitMediaSDK() { RTC_LOG(LS_INFO) << "Encoder NumFrameSuggested=" << alloc_request_.NumFrameSuggested; - // Initialize the Media SDK encoder - sts = encoder_->Init(¶m); - //MSDK_IGNORE_MFX_STS(sts, MFX_WRN_PARTIAL_ACCELERATION); - MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); - - // Retrieve video parameters selected by encoder. - // - BufferSizeInKB parameter is required to set bit stream buffer size - memset(¶m, 0, sizeof(param)); - sts = encoder_->GetVideoParam(¶m); - MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts); - - RTC_LOG(LS_INFO) << "BufferSizeInKB=" << param.mfx.BufferSizeInKB; - frame_info_ = param.mfx.FrameInfo; // 出力ビットストリームの初期化 @@ -512,5 +494,9 @@ int32_t MsdkVideoEncoder::InitMediaSDK() { return WEBRTC_VIDEO_CODEC_OK; } int32_t MsdkVideoEncoder::ReleaseMediaSDK() { + if (encoder_ != nullptr) { + encoder_->Close(); + } + encoder_.reset(); return WEBRTC_VIDEO_CODEC_OK; } \ No newline at end of file diff --git a/src/hwenc_msdk/msdk_video_encoder.h b/src/hwenc_msdk/msdk_video_encoder.h index 7190400a..d0a3c3c2 100644 --- a/src/hwenc_msdk/msdk_video_encoder.h +++ b/src/hwenc_msdk/msdk_video_encoder.h @@ -16,18 +16,18 @@ #include #include -#include "vaapi_utils_drm.h" +#include "msdk_session.h" class MsdkVideoEncoder : public webrtc::VideoEncoder { public: - explicit MsdkVideoEncoder(const cricket::VideoCodec& codec); + explicit MsdkVideoEncoder(std::shared_ptr session, mfxU32 codec); ~MsdkVideoEncoder() override; // MFX_CODEC_VP8 // MFX_CODEC_VP9 // MFX_CODEC_AVC // MFX_CODEC_AV1 - static bool IsSupported(mfxU32 codec); + static bool IsSupported(std::shared_ptr session, mfxU32 codec); int32_t InitEncode(const webrtc::VideoCodec* codec_settings, int32_t number_of_cores, @@ -42,6 +42,16 @@ class MsdkVideoEncoder : public webrtc::VideoEncoder { webrtc::VideoEncoder::EncoderInfo GetEncoderInfo() const override; private: + static std::unique_ptr CreateEncoder( + std::shared_ptr session, + mfxU32 codec, + int width, + int height, + int framerate, + int target_kbps, + int max_kbps, + bool init); + std::mutex mutex_; webrtc::EncodedImageCallback* callback_ = nullptr; webrtc::BitrateAdjuster bitrate_adjuster_; @@ -63,8 +73,8 @@ class MsdkVideoEncoder : public webrtc::VideoEncoder { std::vector surface_buffer_; std::vector surfaces_; - std::unique_ptr libva_; - MFXVideoSession session_; + std::shared_ptr session_; + mfxU32 codec_; mfxFrameAllocRequest alloc_request_; std::unique_ptr encoder_; std::vector bitstream_buffer_; diff --git a/src/main.cpp b/src/main.cpp index b62dc4a5..dd3854fb 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -94,6 +94,14 @@ int main(int argc, char* argv[]) { } #endif +#if USE_MSDK_ENCODER + std::shared_ptr msdk_session; + try { + msdk_session = MsdkSession::Create(); + } catch (...) { + } +#endif + auto capturer = ([&]() -> rtc::scoped_refptr { if (args.no_video_device) { return nullptr; @@ -198,6 +206,10 @@ int main(int argc, char* argv[]) { rtcm_config.cuda_context = cuda_context; #endif +#if USE_MSDK_ENCODER + rtcm_config.msdk_session = msdk_session; +#endif + #if USE_SDL2 std::unique_ptr sdl_renderer = nullptr; if (args.use_sdl) { diff --git a/src/rtc/momo_video_decoder_factory.cpp b/src/rtc/momo_video_decoder_factory.cpp index 517d80cb..0be9abfc 100644 --- a/src/rtc/momo_video_decoder_factory.cpp +++ b/src/rtc/momo_video_decoder_factory.cpp @@ -21,10 +21,16 @@ #if USE_JETSON_ENCODER #include "hwenc_jetson/jetson_video_decoder.h" -#elif USE_MMAL_ENCODER +#endif + +#if USE_MMAL_ENCODER #include "hwenc_mmal/mmal_h264_decoder.h" #endif +#if USE_MSDK_ENCODER +#include "hwenc_msdk/msdk_video_decoder.h" +#endif + namespace { bool IsFormatSupported( @@ -41,51 +47,38 @@ bool IsFormatSupported( } // namespace MomoVideoDecoderFactory::MomoVideoDecoderFactory( - VideoCodecInfo::Type vp8_decoder, - VideoCodecInfo::Type vp9_decoder, - VideoCodecInfo::Type av1_decoder, - VideoCodecInfo::Type h264_decoder -#if USE_NVCODEC_ENCODER - , - std::shared_ptr cuda_context -#endif - ) - : vp8_decoder_(vp8_decoder), - vp9_decoder_(vp9_decoder), - av1_decoder_(av1_decoder), - h264_decoder_(h264_decoder) -#if USE_NVCODEC_ENCODER - , - cuda_context_(cuda_context) -#endif -{ + const MomoVideoDecoderFactoryConfig& config) + : config_(config) { #if defined(__APPLE__) video_decoder_factory_ = CreateObjCDecoderFactory(); #endif } + std::vector MomoVideoDecoderFactory::GetSupportedFormats() const { std::vector supported_codecs; // VP8 - if (vp8_decoder_ == VideoCodecInfo::Type::Software || - vp8_decoder_ == VideoCodecInfo::Type::Jetson || - vp8_decoder_ == VideoCodecInfo::Type::NVIDIA) { + if (config_.vp8_decoder == VideoCodecInfo::Type::Software || + config_.vp8_decoder == VideoCodecInfo::Type::Jetson || + config_.vp8_decoder == VideoCodecInfo::Type::NVIDIA || + config_.vp8_decoder == VideoCodecInfo::Type::Intel) { supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName)); } // VP9 - if (vp9_decoder_ == VideoCodecInfo::Type::Software || - vp9_decoder_ == VideoCodecInfo::Type::Jetson || - vp9_decoder_ == VideoCodecInfo::Type::NVIDIA) { + if (config_.vp9_decoder == VideoCodecInfo::Type::Software || + config_.vp9_decoder == VideoCodecInfo::Type::Jetson || + config_.vp9_decoder == VideoCodecInfo::Type::NVIDIA || + config_.vp9_decoder == VideoCodecInfo::Type::Intel) { for (const webrtc::SdpVideoFormat& format : webrtc::SupportedVP9Codecs()) { supported_codecs.push_back(format); } } // AV1 - // 今のところ Software のみ - if (av1_decoder_ == VideoCodecInfo::Type::Software) { + if (config_.av1_decoder == VideoCodecInfo::Type::Software || + config_.av1_decoder == VideoCodecInfo::Type::Intel) { supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kAv1CodecName)); } @@ -100,14 +93,14 @@ MomoVideoDecoderFactory::GetSupportedFormats() const { CreateH264Format(webrtc::H264Profile::kProfileConstrainedBaseline, webrtc::H264Level::kLevel3_1, "0")}; - if (h264_decoder_ == VideoCodecInfo::Type::VideoToolbox) { + if (config_.h264_decoder == VideoCodecInfo::Type::VideoToolbox) { // VideoToolbox の場合は video_decoder_factory_ から H264 を拾ってくる for (auto format : video_decoder_factory_->GetSupportedFormats()) { if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName)) { supported_codecs.push_back(format); } } - } else if (h264_decoder_ != VideoCodecInfo::Type::NotSupported) { + } else if (config_.h264_decoder != VideoCodecInfo::Type::NotSupported) { // その他のデコーダの場合は手動で追加 for (const webrtc::SdpVideoFormat& h264_format : h264_codecs) { supported_codecs.push_back(h264_format); @@ -127,47 +120,68 @@ MomoVideoDecoderFactory::CreateVideoDecoder( if (absl::EqualsIgnoreCase(format.name, cricket::kVp8CodecName)) { #if USE_NVCODEC_ENCODER - if (vp8_decoder_ == VideoCodecInfo::Type::NVIDIA) { + if (config_.vp8_decoder == VideoCodecInfo::Type::NVIDIA) { return std::unique_ptr( - absl::make_unique(cuda_context_, + absl::make_unique(config_.cuda_context, CudaVideoCodec::VP8)); } #endif +#if USE_MSDK_ENCODER + if (config_.vp8_decoder == VideoCodecInfo::Type::Intel) { + return std::unique_ptr( + absl::make_unique(config_.msdk_session, + MFX_CODEC_VP8)); + } +#endif #if USE_JETSON_ENCODER - if (vp8_decoder_ == VideoCodecInfo::Type::Jetson) { + if (config_.vp8_decoder == VideoCodecInfo::Type::Jetson) { return std::unique_ptr( absl::make_unique(V4L2_PIX_FMT_VP8)); } #endif - if (vp8_decoder_ == VideoCodecInfo::Type::Software) { + if (config_.vp8_decoder == VideoCodecInfo::Type::Software) { return webrtc::VP8Decoder::Create(); } } if (absl::EqualsIgnoreCase(format.name, cricket::kVp9CodecName)) { #if USE_NVCODEC_ENCODER - if (vp9_decoder_ == VideoCodecInfo::Type::NVIDIA) { + if (config_.vp9_decoder == VideoCodecInfo::Type::NVIDIA) { return std::unique_ptr( - absl::make_unique(cuda_context_, + absl::make_unique(config_.cuda_context, CudaVideoCodec::VP9)); } #endif +#if USE_MSDK_ENCODER + if (config_.vp9_decoder == VideoCodecInfo::Type::Intel) { + return std::unique_ptr( + absl::make_unique(config_.msdk_session, + MFX_CODEC_VP9)); + } +#endif #if USE_JETSON_ENCODER - if (vp9_decoder_ == VideoCodecInfo::Type::Jetson) { + if (config_.vp9_decoder == VideoCodecInfo::Type::Jetson) { return std::unique_ptr( absl::make_unique(V4L2_PIX_FMT_VP9)); } #endif - if (vp9_decoder_ == VideoCodecInfo::Type::Software) { + if (config_.vp9_decoder == VideoCodecInfo::Type::Software) { return webrtc::VP9Decoder::Create(); } } if (absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName)) { +#if USE_MSDK_ENCODER + if (config_.av1_decoder == VideoCodecInfo::Type::Intel) { + return std::unique_ptr( + absl::make_unique(config_.msdk_session, + MFX_CODEC_AV1)); + } +#endif #if !defined(__arm__) || defined(__aarch64__) || defined(__ARM_NEON__) - if (av1_decoder_ == VideoCodecInfo::Type::Software) { + if (config_.av1_decoder == VideoCodecInfo::Type::Software) { return webrtc::CreateLibaomAv1Decoder(); } #endif @@ -175,27 +189,34 @@ MomoVideoDecoderFactory::CreateVideoDecoder( if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName)) { #if defined(__APPLE__) - if (h264_decoder_ == VideoCodecInfo::Type::VideoToolbox) { + if (config_.h264_decoder == VideoCodecInfo::Type::VideoToolbox) { return video_decoder_factory_->CreateVideoDecoder(format); } #endif #if USE_NVCODEC_ENCODER - if (h264_decoder_ == VideoCodecInfo::Type::NVIDIA) { + if (config_.h264_decoder == VideoCodecInfo::Type::NVIDIA) { return std::unique_ptr( - absl::make_unique(cuda_context_, + absl::make_unique(config_.cuda_context, CudaVideoCodec::H264)); } #endif +#if USE_MSDK_ENCODER + if (config_.h264_decoder == VideoCodecInfo::Type::Intel) { + return std::unique_ptr( + absl::make_unique(config_.msdk_session, + MFX_CODEC_AVC)); + } +#endif #if USE_JETSON_ENCODER - if (h264_decoder_ == VideoCodecInfo::Type::Jetson) { + if (config_.h264_decoder == VideoCodecInfo::Type::Jetson) { return std::unique_ptr( absl::make_unique(V4L2_PIX_FMT_H264)); } #endif #if USE_MMAL_ENCODER - if (h264_decoder_ == VideoCodecInfo::Type::MMAL) { + if (config_.h264_decoder == VideoCodecInfo::Type::MMAL) { return std::unique_ptr( absl::make_unique()); } diff --git a/src/rtc/momo_video_decoder_factory.h b/src/rtc/momo_video_decoder_factory.h index b0c5ed59..8480c569 100644 --- a/src/rtc/momo_video_decoder_factory.h +++ b/src/rtc/momo_video_decoder_factory.h @@ -12,33 +12,35 @@ #if USE_NVCODEC_ENCODER #include "cuda/cuda_context.h" #endif +#if USE_MSDK_ENCODER +#include "hwenc_msdk/msdk_session.h" +#endif + +struct MomoVideoDecoderFactoryConfig { + VideoCodecInfo::Type vp8_decoder; + VideoCodecInfo::Type vp9_decoder; + VideoCodecInfo::Type av1_decoder; + VideoCodecInfo::Type h264_decoder; +#if USE_NVCODEC_ENCODER + std::shared_ptr cuda_context; +#endif +#if USE_MSDK_ENCODER + std::shared_ptr msdk_session; +#endif +}; class MomoVideoDecoderFactory : public webrtc::VideoDecoderFactory { - VideoCodecInfo::Type vp8_decoder_; - VideoCodecInfo::Type vp9_decoder_; - VideoCodecInfo::Type av1_decoder_; - VideoCodecInfo::Type h264_decoder_; + MomoVideoDecoderFactoryConfig config_; std::unique_ptr video_decoder_factory_; public: - MomoVideoDecoderFactory(VideoCodecInfo::Type vp8_decoder, - VideoCodecInfo::Type vp9_decoder, - VideoCodecInfo::Type av1_decoder, - VideoCodecInfo::Type h264_decoder -#if USE_NVCODEC_ENCODER - , - std::shared_ptr cuda_context -#endif - ); + MomoVideoDecoderFactory(const MomoVideoDecoderFactoryConfig& config); virtual ~MomoVideoDecoderFactory() {} std::vector GetSupportedFormats() const override; std::unique_ptr CreateVideoDecoder( const webrtc::SdpVideoFormat& format) override; -#if USE_NVCODEC_ENCODER - std::shared_ptr cuda_context_; -#endif }; #endif // MOMO_VIDEO_DECODER_FACTORY_H_ diff --git a/src/rtc/momo_video_encoder_factory.cpp b/src/rtc/momo_video_encoder_factory.cpp index c246b716..bcbf8a56 100644 --- a/src/rtc/momo_video_encoder_factory.cpp +++ b/src/rtc/momo_video_encoder_factory.cpp @@ -32,41 +32,20 @@ #if USE_NVCODEC_ENCODER #include "hwenc_nvcodec/nvcodec_h264_encoder.h" #endif +#if USE_MSDK_ENCODER +#include "hwenc_msdk/msdk_video_encoder.h" +#endif MomoVideoEncoderFactory::MomoVideoEncoderFactory( - VideoCodecInfo::Type vp8_encoder, - VideoCodecInfo::Type vp9_encoder, - VideoCodecInfo::Type av1_encoder, - VideoCodecInfo::Type h264_encoder, - bool simulcast, - bool hardware_encoder_only -#if defined(__linux__) && USE_NVCODEC_ENCODER - , - std::shared_ptr cuda_context -#endif - ) - : vp8_encoder_(vp8_encoder), - vp9_encoder_(vp9_encoder), - av1_encoder_(av1_encoder), - h264_encoder_(h264_encoder), - hardware_encoder_only_(hardware_encoder_only) -#if defined(__linux__) && USE_NVCODEC_ENCODER - , - cuda_context_(cuda_context) -#endif -{ + const MomoVideoEncoderFactoryConfig& config) + : config_(config) { #if defined(__APPLE__) video_encoder_factory_ = CreateObjCEncoderFactory(); #endif - if (simulcast) { - internal_encoder_factory_.reset(new MomoVideoEncoderFactory( - vp8_encoder, vp9_encoder, av1_encoder, h264_encoder, false, - hardware_encoder_only -#if defined(__linux__) && USE_NVCODEC_ENCODER - , - cuda_context_ -#endif - )); + if (config.simulcast) { + auto config2 = config; + config2.simulcast = false; + internal_encoder_factory_.reset(new MomoVideoEncoderFactory(config2)); } } std::vector @@ -74,28 +53,28 @@ MomoVideoEncoderFactory::GetSupportedFormats() const { std::vector supported_codecs; // VP8 - if (vp8_encoder_ == VideoCodecInfo::Type::Software || - vp8_encoder_ == VideoCodecInfo::Type::Jetson) { + if (config_.vp8_encoder == VideoCodecInfo::Type::Software || + config_.vp8_encoder == VideoCodecInfo::Type::Jetson || + config_.vp8_encoder == VideoCodecInfo::Type::Intel) { supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName)); } // VP9 - if (vp9_encoder_ == VideoCodecInfo::Type::Software) { + if (config_.vp9_encoder == VideoCodecInfo::Type::Software) { for (const webrtc::SdpVideoFormat& format : webrtc::SupportedVP9Codecs()) { supported_codecs.push_back(format); } - } else if (vp9_encoder_ == VideoCodecInfo::Type::Jetson) { -#if USE_JETSON_ENCODER + } else if (config_.vp9_encoder == VideoCodecInfo::Type::Jetson || + config_.vp9_encoder == VideoCodecInfo::Type::Intel) { supported_codecs.push_back(webrtc::SdpVideoFormat( cricket::kVp9CodecName, {{webrtc::kVP9FmtpProfileId, webrtc::VP9ProfileToString(webrtc::VP9Profile::kProfile0)}})); -#endif } // AV1 - // 今のところ Software のみ - if (av1_encoder_ == VideoCodecInfo::Type::Software) { + if (config_.av1_encoder == VideoCodecInfo::Type::Software || + config_.av1_encoder == VideoCodecInfo::Type::Intel) { supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kAv1CodecName)); } @@ -110,14 +89,14 @@ MomoVideoEncoderFactory::GetSupportedFormats() const { CreateH264Format(webrtc::H264Profile::kProfileConstrainedBaseline, webrtc::H264Level::kLevel3_1, "0")}; - if (h264_encoder_ == VideoCodecInfo::Type::VideoToolbox) { + if (config_.h264_encoder == VideoCodecInfo::Type::VideoToolbox) { // VideoToolbox の場合は video_encoder_factory_ から H264 を拾ってくる for (auto format : video_encoder_factory_->GetSupportedFormats()) { if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName)) { supported_codecs.push_back(format); } } - } else if (h264_encoder_ == VideoCodecInfo::Type::NVIDIA) { + } else if (config_.h264_encoder == VideoCodecInfo::Type::NVIDIA) { #if USE_NVCODEC_ENCODER // NVIDIA の場合は対応してる場合のみ追加 if (NvCodecH264Encoder::IsSupported()) { @@ -126,8 +105,17 @@ MomoVideoEncoderFactory::GetSupportedFormats() const { } } #endif - } else if ((h264_encoder_ == VideoCodecInfo::Type::Software) || - h264_encoder_ != VideoCodecInfo::Type::NotSupported) { + } else if (config_.h264_encoder == VideoCodecInfo::Type::Intel) { +#if USE_MSDK_ENCODER + // Intel Media SDK の場合は対応してる場合のみ追加 + if (MsdkVideoEncoder::IsSupported(config_.msdk_session, MFX_CODEC_AVC)) { + for (const webrtc::SdpVideoFormat& format : h264_codecs) { + supported_codecs.push_back(format); + } + } +#endif + } else if ((config_.h264_encoder == VideoCodecInfo::Type::Software) || + config_.h264_encoder != VideoCodecInfo::Type::NotSupported) { // その他のエンコーダの場合は手動で追加 for (const webrtc::SdpVideoFormat& format : h264_codecs) { supported_codecs.push_back(format); @@ -140,23 +128,23 @@ MomoVideoEncoderFactory::GetSupportedFormats() const { std::unique_ptr MomoVideoEncoderFactory::CreateVideoEncoder( const webrtc::SdpVideoFormat& format) { - // hardware_encoder_only_ == true の場合、Software なコーデックだったら強制終了する - if (hardware_encoder_only_) { + // hardware_encoder_only == true の場合、Software なコーデックだったら強制終了する + if (config_.hardware_encoder_only) { bool use_software = false; if (absl::EqualsIgnoreCase(format.name, cricket::kVp8CodecName) && - vp8_encoder_ == VideoCodecInfo::Type::Software) { + config_.vp8_encoder == VideoCodecInfo::Type::Software) { use_software = true; } if (absl::EqualsIgnoreCase(format.name, cricket::kVp9CodecName) && - vp9_encoder_ == VideoCodecInfo::Type::Software) { + config_.vp9_encoder == VideoCodecInfo::Type::Software) { use_software = true; } if (absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName) && - av1_encoder_ == VideoCodecInfo::Type::Software) { + config_.av1_encoder == VideoCodecInfo::Type::Software) { use_software = true; } if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName) && - h264_encoder_ == VideoCodecInfo::Type::Software) { + config_.h264_encoder == VideoCodecInfo::Type::Software) { use_software = true; } @@ -175,50 +163,77 @@ MomoVideoEncoderFactory::CreateVideoEncoder( } if (absl::EqualsIgnoreCase(format.name, cricket::kVp8CodecName)) { - if (vp8_encoder_ == VideoCodecInfo::Type::Software) { + if (config_.vp8_encoder == VideoCodecInfo::Type::Software) { return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return webrtc::VP8Encoder::Create(); }); } #if USE_JETSON_ENCODER - if (vp8_encoder_ == VideoCodecInfo::Type::Jetson) { + if (config_.vp8_encoder == VideoCodecInfo::Type::Jetson) { return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return std::unique_ptr( absl::make_unique(cricket::VideoCodec(format))); }); } +#endif +#if USE_MSDK_ENCODER + if (config_.vp8_encoder == VideoCodecInfo::Type::Intel) { + return WithSimulcast(format, [msdk_session = config_.msdk_session]( + const webrtc::SdpVideoFormat& format) { + return std::unique_ptr( + absl::make_unique(msdk_session, MFX_CODEC_VP8)); + }); + } #endif } if (absl::EqualsIgnoreCase(format.name, cricket::kVp9CodecName)) { - if (vp9_encoder_ == VideoCodecInfo::Type::Software) { + if (config_.vp9_encoder == VideoCodecInfo::Type::Software) { return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return webrtc::VP9Encoder::Create(cricket::VideoCodec(format)); }); } #if USE_JETSON_ENCODER - if (vp9_encoder_ == VideoCodecInfo::Type::Jetson) { + if (config_.vp9_encoder == VideoCodecInfo::Type::Jetson) { return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return std::unique_ptr( absl::make_unique(cricket::VideoCodec(format))); }); } +#endif +#if USE_MSDK_ENCODER + if (config_.vp9_encoder == VideoCodecInfo::Type::Intel) { + return WithSimulcast(format, [msdk_session = config_.msdk_session]( + const webrtc::SdpVideoFormat& format) { + return std::unique_ptr( + absl::make_unique(msdk_session, MFX_CODEC_VP9)); + }); + } #endif } if (absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName)) { #if !defined(__arm__) || defined(__aarch64__) || defined(__ARM_NEON__) - if (av1_encoder_ == VideoCodecInfo::Type::Software) { + if (config_.av1_encoder == VideoCodecInfo::Type::Software) { return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return webrtc::CreateLibaomAv1Encoder(); }); } +#endif +#if USE_MSDK_ENCODER + if (config_.av1_encoder == VideoCodecInfo::Type::Intel) { + return WithSimulcast(format, [msdk_session = config_.msdk_session]( + const webrtc::SdpVideoFormat& format) { + return std::unique_ptr( + absl::make_unique(msdk_session, MFX_CODEC_AV1)); + }); + } #endif } if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName)) { #if defined(__APPLE__) - if (h264_encoder_ == VideoCodecInfo::Type::VideoToolbox) { + if (config_.h264_encoder == VideoCodecInfo::Type::VideoToolbox) { return WithSimulcast( format, [this](const webrtc::SdpVideoFormat& format) { return video_encoder_factory_->CreateVideoEncoder(format); @@ -227,7 +242,7 @@ MomoVideoEncoderFactory::CreateVideoEncoder( #endif #if USE_MMAL_ENCODER - if (h264_encoder_ == VideoCodecInfo::Type::MMAL) { + if (config_.h264_encoder == VideoCodecInfo::Type::MMAL) { return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return std::unique_ptr( absl::make_unique(cricket::VideoCodec(format))); @@ -236,7 +251,7 @@ MomoVideoEncoderFactory::CreateVideoEncoder( #endif #if USE_JETSON_ENCODER - if (h264_encoder_ == VideoCodecInfo::Type::Jetson) { + if (config_.h264_encoder == VideoCodecInfo::Type::Jetson) { return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return std::unique_ptr( absl::make_unique(cricket::VideoCodec(format))); @@ -245,23 +260,35 @@ MomoVideoEncoderFactory::CreateVideoEncoder( #endif #if USE_NVCODEC_ENCODER - if (h264_encoder_ == VideoCodecInfo::Type::NVIDIA && + if (config_.h264_encoder == VideoCodecInfo::Type::NVIDIA && NvCodecH264Encoder::IsSupported()) { return WithSimulcast( #if defined(__linux__) format, - [cuda_context = cuda_context_](const webrtc::SdpVideoFormat& format) { + [cuda_context = + config_.cuda_context](const webrtc::SdpVideoFormat& format) { return std::unique_ptr( absl::make_unique( cricket::VideoCodec(format), cuda_context)); + } #else format, [](const webrtc::SdpVideoFormat& format) { return std::unique_ptr( absl::make_unique( cricket::VideoCodec(format))); + } #endif - }); + ); + } +#endif +#if USE_MSDK_ENCODER + if (config_.h264_encoder == VideoCodecInfo::Type::Intel) { + return WithSimulcast(format, [msdk_session = config_.msdk_session]( + const webrtc::SdpVideoFormat& format) { + return std::unique_ptr( + absl::make_unique(msdk_session, MFX_CODEC_AVC)); + }); } #endif } diff --git a/src/rtc/momo_video_encoder_factory.h b/src/rtc/momo_video_encoder_factory.h index cd007c5a..45846b3a 100644 --- a/src/rtc/momo_video_encoder_factory.h +++ b/src/rtc/momo_video_encoder_factory.h @@ -15,27 +15,32 @@ #include "cuda/cuda_context.h" #endif +#if USE_MSDK_ENCODER +#include "hwenc_msdk/msdk_session.h" +#endif + +struct MomoVideoEncoderFactoryConfig { + VideoCodecInfo::Type vp8_encoder; + VideoCodecInfo::Type vp9_encoder; + VideoCodecInfo::Type av1_encoder; + VideoCodecInfo::Type h264_encoder; + bool simulcast; + bool hardware_encoder_only; +#if defined(__linux__) && USE_NVCODEC_ENCODER + std::shared_ptr cuda_context; +#endif +#if USE_MSDK_ENCODER + std::shared_ptr msdk_session; +#endif +}; + class MomoVideoEncoderFactory : public webrtc::VideoEncoderFactory { - VideoCodecInfo::Type vp8_encoder_; - VideoCodecInfo::Type vp9_encoder_; - VideoCodecInfo::Type av1_encoder_; - VideoCodecInfo::Type h264_encoder_; + MomoVideoEncoderFactoryConfig config_; std::unique_ptr video_encoder_factory_; std::unique_ptr internal_encoder_factory_; - bool hardware_encoder_only_; public: - MomoVideoEncoderFactory(VideoCodecInfo::Type vp8_encoder, - VideoCodecInfo::Type vp9_encoder, - VideoCodecInfo::Type av1_encoder, - VideoCodecInfo::Type h264_encoder, - bool simulcast, - bool hardware_encoder_only -#if defined(__linux__) && USE_NVCODEC_ENCODER - , - std::shared_ptr cuda_context -#endif - ); + MomoVideoEncoderFactory(const MomoVideoEncoderFactoryConfig& config); virtual ~MomoVideoEncoderFactory() {} std::vector GetSupportedFormats() const override; @@ -48,9 +53,6 @@ class MomoVideoEncoderFactory : public webrtc::VideoEncoderFactory { const webrtc::SdpVideoFormat& format, std::function( const webrtc::SdpVideoFormat&)> create); -#if defined(__linux__) && USE_NVCODEC_ENCODER - std::shared_ptr cuda_context_; -#endif }; #endif // MOMO_VIDEO_ENCODER_FACTORY_H_ diff --git a/src/rtc/rtc_manager.cpp b/src/rtc/rtc_manager.cpp index 38d4f3ff..f110feb3 100644 --- a/src/rtc/rtc_manager.cpp +++ b/src/rtc/rtc_manager.cpp @@ -96,31 +96,36 @@ RTCManager::RTCManager( // 名前を短くする auto& cf = config_; auto resolve = &VideoCodecInfo::Resolve; + MomoVideoEncoderFactoryConfig ec; + ec.vp8_encoder = resolve(cf.vp8_encoder, info.vp8_encoders); + ec.vp9_encoder = resolve(cf.vp9_encoder, info.vp9_encoders); + ec.av1_encoder = resolve(cf.av1_encoder, info.av1_encoders); + ec.h264_encoder = resolve(cf.h264_encoder, info.h264_encoders); + ec.simulcast = cf.simulcast; + ec.hardware_encoder_only = cf.hardware_encoder_only; +#if defined(__linux__) && USE_NVCODEC_ENCODER + ec.cuda_context = cf.cuda_context; +#endif +#if USE_MSDK_ENCODER + ec.msdk_session = cf.msdk_session; +#endif media_dependencies.video_encoder_factory = std::unique_ptr( - absl::make_unique( - resolve(cf.vp8_encoder, info.vp8_encoders), - resolve(cf.vp9_encoder, info.vp9_encoders), - resolve(cf.av1_encoder, info.av1_encoders), - resolve(cf.h264_encoder, info.h264_encoders), cf.simulcast, - cf.hardware_encoder_only -#if defined(__linux__) && USE_NVCODEC_ENCODER - , - cf.cuda_context + absl::make_unique(ec)); + MomoVideoDecoderFactoryConfig dc; + dc.vp8_decoder = resolve(cf.vp8_decoder, info.vp8_decoders); + dc.vp9_decoder = resolve(cf.vp9_decoder, info.vp9_decoders); + dc.av1_decoder = resolve(cf.av1_decoder, info.av1_decoders); + dc.h264_decoder = resolve(cf.h264_decoder, info.h264_decoders); +#if USE_NVCODEC_ENCODER + dc.cuda_context = cf.cuda_context; +#endif +#if USE_MSDK_ENCODER + dc.msdk_session = cf.msdk_session; #endif - )); media_dependencies.video_decoder_factory = std::unique_ptr( - absl::make_unique( - resolve(cf.vp8_decoder, info.vp8_decoders), - resolve(cf.vp9_decoder, info.vp9_decoders), - resolve(cf.av1_decoder, info.av1_decoders), - resolve(cf.h264_decoder, info.h264_decoders) -#if USE_NVCODEC_ENCODER - , - cf.cuda_context -#endif - )); + absl::make_unique(dc)); } media_dependencies.audio_mixer = nullptr; diff --git a/src/rtc/rtc_manager.h b/src/rtc/rtc_manager.h index 23dd1b79..657c0c24 100644 --- a/src/rtc/rtc_manager.h +++ b/src/rtc/rtc_manager.h @@ -15,6 +15,10 @@ #include "video_codec_info.h" #include "video_track_receiver.h" +#if USE_MSDK_ENCODER +#include "hwenc_msdk/msdk_session.h" +#endif + struct RTCManagerConfig { bool insecure = false; @@ -56,6 +60,9 @@ struct RTCManagerConfig { #if USE_NVCODEC_ENCODER std::shared_ptr cuda_context; #endif +#if USE_MSDK_ENCODER + std::shared_ptr msdk_session; +#endif }; class RTCManager { diff --git a/src/video_codec_info.h b/src/video_codec_info.h index 5a425729..d4326eb1 100644 --- a/src/video_codec_info.h +++ b/src/video_codec_info.h @@ -169,28 +169,29 @@ struct VideoCodecInfo { #endif #if USE_MSDK_ENCODER - if (MsdkVideoEncoder::IsSupported(MFX_CODEC_VP8)) { + auto session = MsdkSession::Create(); + if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_VP8)) { info.vp8_encoders.push_back(Type::Intel); } - if (MsdkVideoEncoder::IsSupported(MFX_CODEC_VP9)) { + if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_VP9)) { info.vp9_encoders.push_back(Type::Intel); } - if (MsdkVideoEncoder::IsSupported(MFX_CODEC_AVC)) { + if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_AVC)) { info.h264_encoders.push_back(Type::Intel); } - if (MsdkVideoEncoder::IsSupported(MFX_CODEC_AV1)) { + if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_AV1)) { info.av1_encoders.push_back(Type::Intel); } - if (MsdkVideoDecoder::IsSupported(MFX_CODEC_VP8)) { + if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_VP8)) { info.vp8_decoders.push_back(Type::Intel); } - if (MsdkVideoDecoder::IsSupported(MFX_CODEC_VP9)) { + if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_VP9)) { info.vp9_decoders.push_back(Type::Intel); } - if (MsdkVideoDecoder::IsSupported(MFX_CODEC_AVC)) { + if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_AVC)) { info.h264_decoders.push_back(Type::Intel); } - if (MsdkVideoDecoder::IsSupported(MFX_CODEC_AV1)) { + if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_AV1)) { info.av1_decoders.push_back(Type::Intel); } #endif From 03db12dc30409e210f696d6f735234b0b41b4e42 Mon Sep 17 00:00:00 2001 From: melpon Date: Fri, 8 Apr 2022 02:19:00 +0900 Subject: [PATCH 39/70] =?UTF-8?q?Intel=20Media=20SDK=20=E3=81=8C=E4=BD=BF?= =?UTF-8?q?=E3=81=88=E3=81=AA=E3=81=84=E6=99=82=E3=81=AB=E8=90=BD=E3=81=A1?= =?UTF-8?q?=E3=81=AA=E3=81=84=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/c_cpp_properties.json | 2 +- build/ubuntu-20.04_x86_64/Dockerfile | 2 +- script/apt_install_x86_64.sh | 5 ++- src/video_codec_info.h | 48 +++++++++++++++------------- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 48afa950..9e7996e8 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -81,7 +81,7 @@ "${workspaceFolder}/_install/raspberry-pi-os_armv7/CLI11/include", "${workspaceFolder}/_install/raspberry-pi-os_armv7/SDL2/include/SDL2", "${workspaceFolder}/_install/raspberry-pi-os_armv7/rootfs/usr/include", - "${workspaceFolder}/_build/raspberry-pi-os_armv7", + "${workspaceFolder}/_build/raspberry-pi-os_armv7" ], "defines": [ "WEBRTC_LINUX", diff --git a/build/ubuntu-20.04_x86_64/Dockerfile b/build/ubuntu-20.04_x86_64/Dockerfile index 6a5e3689..f1acd3a5 100644 --- a/build/ubuntu-20.04_x86_64/Dockerfile +++ b/build/ubuntu-20.04_x86_64/Dockerfile @@ -12,7 +12,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloa COPY script/apt_install_x86_64.sh /root/ RUN --mount=type=cache,id=$PACKAGE_NAME,target=/var/cache/apt --mount=type=cache,id=$PACKAGE_NAME,target=/var/lib/apt \ /root/apt_install_x86_64.sh \ - apt install libtool libdrm-dev + apt install libdrm-dev # WebRTC の取得 diff --git a/script/apt_install_x86_64.sh b/script/apt_install_x86_64.sh index 9e1fd52f..03974adf 100755 --- a/script/apt_install_x86_64.sh +++ b/script/apt_install_x86_64.sh @@ -10,6 +10,8 @@ dpkg-reconfigure -f noninteractive tzdata # libtinfo5 は Ubuntu 20.04 のために入れたが将来的に不要になる可能性がある apt-get -y install \ + autoconf \ + automake \ build-essential \ curl \ git \ @@ -17,10 +19,11 @@ apt-get -y install \ libc6-dev \ libexpat1-dev \ libgtk-3-dev \ - libtinfo5 \ libnspr4-dev \ libnss3-dev \ libpulse-dev \ + libtinfo5 \ + libtool \ libudev-dev \ libxrandr-dev \ lsb-release \ diff --git a/src/video_codec_info.h b/src/video_codec_info.h index d4326eb1..fd1b4ec2 100644 --- a/src/video_codec_info.h +++ b/src/video_codec_info.h @@ -170,29 +170,31 @@ struct VideoCodecInfo { #if USE_MSDK_ENCODER auto session = MsdkSession::Create(); - if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_VP8)) { - info.vp8_encoders.push_back(Type::Intel); - } - if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_VP9)) { - info.vp9_encoders.push_back(Type::Intel); - } - if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_AVC)) { - info.h264_encoders.push_back(Type::Intel); - } - if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_AV1)) { - info.av1_encoders.push_back(Type::Intel); - } - if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_VP8)) { - info.vp8_decoders.push_back(Type::Intel); - } - if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_VP9)) { - info.vp9_decoders.push_back(Type::Intel); - } - if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_AVC)) { - info.h264_decoders.push_back(Type::Intel); - } - if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_AV1)) { - info.av1_decoders.push_back(Type::Intel); + if (session != nullptr) { + if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_VP8)) { + info.vp8_encoders.push_back(Type::Intel); + } + if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_VP9)) { + info.vp9_encoders.push_back(Type::Intel); + } + if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_AVC)) { + info.h264_encoders.push_back(Type::Intel); + } + if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_AV1)) { + info.av1_encoders.push_back(Type::Intel); + } + if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_VP8)) { + info.vp8_decoders.push_back(Type::Intel); + } + if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_VP9)) { + info.vp9_decoders.push_back(Type::Intel); + } + if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_AVC)) { + info.h264_decoders.push_back(Type::Intel); + } + if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_AV1)) { + info.av1_decoders.push_back(Type::Intel); + } } #endif From 32229ddc3af208a92e5ff72340c8b2297497ba11 Mon Sep 17 00:00:00 2001 From: melpon Date: Fri, 8 Apr 2022 02:43:11 +0900 Subject: [PATCH 40/70] =?UTF-8?q?=E3=83=93=E3=83=AB=E3=83=89=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/ubuntu-20.04_x86_64/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ubuntu-20.04_x86_64/Dockerfile b/build/ubuntu-20.04_x86_64/Dockerfile index f1acd3a5..1fb53ef7 100644 --- a/build/ubuntu-20.04_x86_64/Dockerfile +++ b/build/ubuntu-20.04_x86_64/Dockerfile @@ -12,7 +12,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloa COPY script/apt_install_x86_64.sh /root/ RUN --mount=type=cache,id=$PACKAGE_NAME,target=/var/cache/apt --mount=type=cache,id=$PACKAGE_NAME,target=/var/lib/apt \ /root/apt_install_x86_64.sh \ - apt install libdrm-dev + && apt-get -y install libdrm-dev # WebRTC の取得 From 6166c477b9a382604f09b560fa1bbc51d7ee9ec2 Mon Sep 17 00:00:00 2001 From: melpon Date: Fri, 8 Apr 2022 13:55:35 +0900 Subject: [PATCH 41/70] =?UTF-8?q?Windows=20=E3=81=A7=20Intel=20Media=20SDK?= =?UTF-8?q?=20=E3=82=92=E6=9C=89=E5=8A=B9=E3=81=AB=E3=81=97=E3=81=A6?= =?UTF-8?q?=E3=83=93=E3=83=AB=E3=83=89=E3=82=92=E9=80=9A=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/c_cpp_properties.json | 1 + CMakeLists.txt | 16 ++++++++++ build/windows_x86_64/install_deps.ps1 | 22 ++++++++++++++ src/hwenc_msdk/msdk_session.cpp | 44 ++++++++++++++++++++++++--- src/hwenc_msdk/msdk_session.h | 9 ++++++ src/hwenc_msdk/msdk_utils.h | 12 ++++++++ src/hwenc_msdk/msdk_video_decoder.cpp | 5 ++- src/hwenc_msdk/msdk_video_decoder.h | 1 - src/hwenc_msdk/msdk_video_encoder.cpp | 2 ++ src/hwenc_msdk/vaapi_utils.h | 8 ----- 10 files changed, 103 insertions(+), 17 deletions(-) create mode 100644 src/hwenc_msdk/msdk_utils.h diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 9e7996e8..d86ac8bf 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -15,6 +15,7 @@ "${workspaceFolder}/build/windows_x86_64/_install/CLI11/include", "${workspaceFolder}/build/windows_x86_64/_install/SDL2/include/SDL2", "${workspaceFolder}/build/windows_x86_64/_install/cuda/nvcc/include", + "${workspaceFolder}/build/windows_x86_64/_install/msdk/include", "${workspaceFolder}/_build/windows_x86_64" ], "defines": [ diff --git a/CMakeLists.txt b/CMakeLists.txt index a768b2d7..179c1832 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,7 @@ if(MOMO_PACKAGE_NAME STREQUAL "windows_x86_64") set(TARGET_OS "windows") set(TARGET_ARCH "x86_64") set(USE_NVCODEC_ENCODER ON) + set(USE_MSDK_ENCODER ON) set(USE_H264 ON) set(USE_SDL2 ON) set(USE_SCREEN_CAPTURER ON) @@ -552,6 +553,21 @@ if ("${TARGET_OS}" STREQUAL "windows") #${CUDA_LIBRARIES} ) endif() + + # Intel Media SDK + if (USE_MSDK_ENCODER) + target_sources(momo + PRIVATE + src/hwenc_msdk/msdk_session.cpp + src/hwenc_msdk/msdk_video_decoder.cpp + src/hwenc_msdk/msdk_video_encoder.cpp + ) + target_include_directories(momo PRIVATE ${_INSTALL_DIR}/msdk/include) + target_link_libraries(momo + PRIVATE + ${_INSTALL_DIR}/msdk/lib/libmfx.lib + ) + endif() elseif (TARGET_OS STREQUAL "macos") target_sources(momo PRIVATE diff --git a/build/windows_x86_64/install_deps.ps1 b/build/windows_x86_64/install_deps.ps1 index 822c1c76..53b39950 100644 --- a/build/windows_x86_64/install_deps.ps1 +++ b/build/windows_x86_64/install_deps.ps1 @@ -165,4 +165,26 @@ if (!(Test-Path "$INSTALL_DIR\cuda\nvcc")) { mkdir $INSTALL_DIR\cuda Move-Item nvcc $INSTALL_DIR\cuda\nvcc Pop-Location +} + +# Intel Media SDK +if (!(Test-Path "$INSTALL_DIR\msdk\lib\libmfx.lib")) { + if (Test-Path "$SOURCE_DIR\msdk") { + Remove-Item $SOURCE_DIR\msdk -Recurse -Force + } + if (Test-Path "$INSTALL_DIR\msdk") { + Remove-Item $INSTALL_DIR\msdk -Recurse -Force + } + mkdir $SOURCE_DIR\msdk + git clone --depth 1 --branch intel-mediasdk-$MSDK_VERSION https://github.com/Intel-Media-SDK/MediaSDK.git $SOURCE_DIR\msdk\MediaSDK + mkdir $INSTALL_DIR\msdk + mkdir $INSTALL_DIR\msdk\include + mkdir $INSTALL_DIR\msdk\lib + Copy-Item -Recurse $SOURCE_DIR\msdk\MediaSDK\api\include $INSTALL_DIR\msdk\include\mfx + $WSDK_VERSION = $(Get-Item "HKLM:\SOFTWARE\WOW6432Node\Microsoft\Microsoft SDKs\Windows\v10.0").GetValue("ProductVersion") + & "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe" ` + /t:build ` + "/p:Configuration=Release;Platform=x64;PlatformToolset=v142;SpectreMitigation=false;WindowsTargetPlatformVersion=$WSDK_VERSION.0" ` + $SOURCE_DIR\msdk\MediaSDK\api\mfx_dispatch\windows\libmfx_vs2015.vcxproj + Copy-Item $SOURCE_DIR\msdk\build\win_x64\Release\lib\libmfx_vs2015.lib $INSTALL_DIR\msdk\lib\libmfx.lib } \ No newline at end of file diff --git a/src/hwenc_msdk/msdk_session.cpp b/src/hwenc_msdk/msdk_session.cpp index dbdf9521..e1d114c0 100644 --- a/src/hwenc_msdk/msdk_session.cpp +++ b/src/hwenc_msdk/msdk_session.cpp @@ -2,27 +2,61 @@ std::shared_ptr MsdkSession::Create() { std::shared_ptr session(new MsdkSession()); - session->libva = CreateDRMLibVA(); - if (!session->libva) { - return nullptr; - } mfxStatus sts = MFX_ERR_NONE; - mfxIMPL impl = MFX_IMPL_HARDWARE_ANY; mfxVersion ver = {{0, 1}}; +#ifdef __linux__ + mfxIMPL impl = MFX_IMPL_HARDWARE_ANY; sts = session->session.Init(impl, &ver); if (sts != MFX_ERR_NONE) { return nullptr; } + session->libva = CreateDRMLibVA(); + if (!session->libva) { + return nullptr; + } + sts = session->session.SetHandle( static_cast(MFX_HANDLE_VA_DISPLAY), session->libva->GetVADisplay()); if (sts != MFX_ERR_NONE) { return nullptr; } +#endif + +#ifdef _WIN32 + mfxIMPL impl = MFX_IMPL_VIA_D3D11 | MFX_IMPL_HARDWARE_ANY; + + sts = session->session.Init(impl, &ver); + if (sts != MFX_ERR_NONE) { + return nullptr; + } + + Microsoft::WRL::ComPtr idxgi_factory; + if (FAILED(CreateDXGIFactory1(__uuidof(IDXGIFactory1), + (void**)idxgi_factory.GetAddressOf()))) { + return nullptr; + } + Microsoft::WRL::ComPtr idxgi_adapter; + if (FAILED(idxgi_factory->EnumAdapters(0, idxgi_adapter.GetAddressOf()))) { + return nullptr; + } + if (FAILED(D3D11CreateDevice(idxgi_adapter.Get(), D3D_DRIVER_TYPE_UNKNOWN, + NULL, 0, NULL, 0, D3D11_SDK_VERSION, + session->d3d11_device.GetAddressOf(), NULL, + session->d3d11_context.GetAddressOf()))) { + return nullptr; + } + + sts = session->session.SetHandle(MFX_HANDLE_D3D11_DEVICE, + session->d3d11_device.Get()); + if (sts != MFX_ERR_NONE) { + return nullptr; + } +#endif // Query selected implementation and version sts = session->session.QueryIMPL(&impl); diff --git a/src/hwenc_msdk/msdk_session.h b/src/hwenc_msdk/msdk_session.h index 5dfab113..056705e9 100644 --- a/src/hwenc_msdk/msdk_session.h +++ b/src/hwenc_msdk/msdk_session.h @@ -10,12 +10,21 @@ #include "vaapi_utils_drm.h" #endif +#ifdef _WIN32 +#include +#include +#endif + struct MsdkSession { MFXVideoSession session; #ifdef __linux__ std::unique_ptr libva; #endif +#ifdef _WIN32 + Microsoft::WRL::ComPtr d3d11_device; + Microsoft::WRL::ComPtr d3d11_context; +#endif static std::shared_ptr Create(); }; diff --git a/src/hwenc_msdk/msdk_utils.h b/src/hwenc_msdk/msdk_utils.h new file mode 100644 index 00000000..8ddeeadc --- /dev/null +++ b/src/hwenc_msdk/msdk_utils.h @@ -0,0 +1,12 @@ +#ifndef MSDK_UTILS_H_ +#define MSDK_UTILS_H_ + +#define MSDK_CHECK_RESULT(P, X, ERR) \ + { \ + if ((X) > (P)) { \ + RTC_LOG(LS_ERROR) << "Intel Media SDK Error: " << ERR; \ + throw ERR; \ + } \ + } + +#endif \ No newline at end of file diff --git a/src/hwenc_msdk/msdk_video_decoder.cpp b/src/hwenc_msdk/msdk_video_decoder.cpp index a96b017f..abe02c51 100644 --- a/src/hwenc_msdk/msdk_video_decoder.cpp +++ b/src/hwenc_msdk/msdk_video_decoder.cpp @@ -3,9 +3,6 @@ #include #include -// Linux -#include - // WebRTC #include #include @@ -13,6 +10,8 @@ #include #include +#include "msdk_utils.h" + MsdkVideoDecoder::MsdkVideoDecoder(std::shared_ptr session, mfxU32 codec) : session_(session), diff --git a/src/hwenc_msdk/msdk_video_decoder.h b/src/hwenc_msdk/msdk_video_decoder.h index 51187a90..f4bc9052 100644 --- a/src/hwenc_msdk/msdk_video_decoder.h +++ b/src/hwenc_msdk/msdk_video_decoder.h @@ -12,7 +12,6 @@ #include #include "msdk_session.h" -#include "vaapi_utils_drm.h" class MsdkVideoDecoder : public webrtc::VideoDecoder { public: diff --git a/src/hwenc_msdk/msdk_video_encoder.cpp b/src/hwenc_msdk/msdk_video_encoder.cpp index 6e272519..cee3f05d 100644 --- a/src/hwenc_msdk/msdk_video_encoder.cpp +++ b/src/hwenc_msdk/msdk_video_encoder.cpp @@ -5,6 +5,8 @@ // libyuv #include +#include "msdk_utils.h" + const int kLowH264QpThreshold = 34; const int kHighH264QpThreshold = 40; diff --git a/src/hwenc_msdk/vaapi_utils.h b/src/hwenc_msdk/vaapi_utils.h index 522f0a86..ad9dd3a6 100644 --- a/src/hwenc_msdk/vaapi_utils.h +++ b/src/hwenc_msdk/vaapi_utils.h @@ -41,12 +41,4 @@ class CLibVA { mfxStatus va_to_mfx_status(VAStatus va_res); -#define MSDK_CHECK_RESULT(P, X, ERR) \ - { \ - if ((X) > (P)) { \ - RTC_LOG(LS_ERROR) << "Intel Media SDK Error: " << ERR; \ - throw ERR; \ - } \ - } - #endif // VAAPI_UTILS_H_ \ No newline at end of file From a78c19c9d86f5503cab004eb21307efd070227be Mon Sep 17 00:00:00 2001 From: melpon Date: Fri, 8 Apr 2022 14:19:18 +0900 Subject: [PATCH 42/70] =?UTF-8?q?MSBuild=20=E3=81=AE=E3=83=91=E3=82=B9?= =?UTF-8?q?=E3=82=92=E6=8C=87=E5=AE=9A=E3=81=97=E3=81=AA=E3=81=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/windows_x86_64/install_deps.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/windows_x86_64/install_deps.ps1 b/build/windows_x86_64/install_deps.ps1 index 53b39950..7c6cbde7 100644 --- a/build/windows_x86_64/install_deps.ps1 +++ b/build/windows_x86_64/install_deps.ps1 @@ -182,7 +182,7 @@ if (!(Test-Path "$INSTALL_DIR\msdk\lib\libmfx.lib")) { mkdir $INSTALL_DIR\msdk\lib Copy-Item -Recurse $SOURCE_DIR\msdk\MediaSDK\api\include $INSTALL_DIR\msdk\include\mfx $WSDK_VERSION = $(Get-Item "HKLM:\SOFTWARE\WOW6432Node\Microsoft\Microsoft SDKs\Windows\v10.0").GetValue("ProductVersion") - & "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe" ` + MSBuild ` /t:build ` "/p:Configuration=Release;Platform=x64;PlatformToolset=v142;SpectreMitigation=false;WindowsTargetPlatformVersion=$WSDK_VERSION.0" ` $SOURCE_DIR\msdk\MediaSDK\api\mfx_dispatch\windows\libmfx_vs2015.vcxproj From 532be7ca55a4525180d7edd05998642e0aaa7e7f Mon Sep 17 00:00:00 2001 From: melpon Date: Fri, 8 Apr 2022 19:04:30 +0900 Subject: [PATCH 43/70] =?UTF-8?q?setup-msbuild=20=E3=81=97=E3=81=A6?= =?UTF-8?q?=E3=80=81=E3=83=AD=E3=83=BC=E3=82=AB=E3=83=AB=E3=81=A7=E3=82=82?= =?UTF-8?q?=E3=83=93=E3=83=AB=E3=83=89=E3=81=8C=E9=80=9A=E3=82=8B=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 1 + build/windows_x86_64/install_deps.ps1 | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e9548941..9f559c47 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,6 +21,7 @@ jobs: runs-on: windows-2019 steps: - uses: actions/checkout@v3 + - uses: microsoft/setup-msbuild@v1.1 - name: Get Versions run: | Get-Content "VERSION" | Foreach-Object { diff --git a/build/windows_x86_64/install_deps.ps1 b/build/windows_x86_64/install_deps.ps1 index 7c6cbde7..ad5c4a82 100644 --- a/build/windows_x86_64/install_deps.ps1 +++ b/build/windows_x86_64/install_deps.ps1 @@ -182,9 +182,16 @@ if (!(Test-Path "$INSTALL_DIR\msdk\lib\libmfx.lib")) { mkdir $INSTALL_DIR\msdk\lib Copy-Item -Recurse $SOURCE_DIR\msdk\MediaSDK\api\include $INSTALL_DIR\msdk\include\mfx $WSDK_VERSION = $(Get-Item "HKLM:\SOFTWARE\WOW6432Node\Microsoft\Microsoft SDKs\Windows\v10.0").GetValue("ProductVersion") - MSBuild ` - /t:build ` - "/p:Configuration=Release;Platform=x64;PlatformToolset=v142;SpectreMitigation=false;WindowsTargetPlatformVersion=$WSDK_VERSION.0" ` - $SOURCE_DIR\msdk\MediaSDK\api\mfx_dispatch\windows\libmfx_vs2015.vcxproj + if (Test-Path "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe") { + & "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe" ` + /t:build ` + "/p:Configuration=Release;Platform=x64;PlatformToolset=v142;SpectreMitigation=false;WindowsTargetPlatformVersion=$WSDK_VERSION.0" ` + $SOURCE_DIR\msdk\MediaSDK\api\mfx_dispatch\windows\libmfx_vs2015.vcxproj + } else { + MSBuild ` + /t:build ` + "/p:Configuration=Release;Platform=x64;PlatformToolset=v142;SpectreMitigation=false;WindowsTargetPlatformVersion=$WSDK_VERSION.0" ` + $SOURCE_DIR\msdk\MediaSDK\api\mfx_dispatch\windows\libmfx_vs2015.vcxproj + } Copy-Item $SOURCE_DIR\msdk\build\win_x64\Release\lib\libmfx_vs2015.lib $INSTALL_DIR\msdk\lib\libmfx.lib } \ No newline at end of file From bf05caed7d5821716bb9f4183ccf98a64c3c2ed2 Mon Sep 17 00:00:00 2001 From: melpon Date: Fri, 8 Apr 2022 20:01:01 +0900 Subject: [PATCH 44/70] =?UTF-8?q?CHANGES=20=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 83f111aa..0bd8972d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,8 @@ - @melpon - [ADD] Windows と Ubuntu で NVIDIA VIDEO CODEC SDK を使ったハードウェアデコーダに対応 - @melpon +- [ADD] Windows と Ubuntu 20.04 で Intel Media SDK に対応 + - @melpon - [UPDATE] CLI11 を 2.2.0 に上げる - @voluntas - [FIX] Ubuntu 20.04 + H.264 + サイマルキャスト + --hw-mjpeg-decoder true で落ちるのを修正 (#221) From a4f2ef34c1a543fe7b169d4653d4a9706d89366a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Apr 2022 14:31:39 +0000 Subject: [PATCH 45/70] Bump actions/upload-artifact from 2 to 3 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 2 to 3. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9f559c47..0256c0d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,12 +65,12 @@ jobs: echo "::set-output name=name::momo-${MOMO_VERSION}_windows-${WINVER_MAJOR}.${RELEASE_ID}_${{ matrix.arch }}.zip" id: package_name - name: Upload Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ${{ steps.package_name.outputs.name }} path: _package/${{ steps.package_name.outputs.name }} - name: Upload Environment - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: windows_${{ matrix.arch }}.env path: windows_${{ matrix.arch }}.env @@ -109,12 +109,12 @@ jobs: echo "::set-output name=name::momo-${MOMO_VERSION}_macos-${MACOS_VERSION}_${{ matrix.arch }}.tar.gz" id: package_name - name: Upload Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ${{ steps.package_name.outputs.name }} path: _package/${{ steps.package_name.outputs.name }} - name: Upload Environment - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: macos_${{ matrix.arch }}.env path: macos_${{ matrix.arch }}.env @@ -157,12 +157,12 @@ jobs: echo "::set-output name=name::momo-${MOMO_VERSION}_${{ matrix.name }}.tar.gz" id: package_name - name: Upload Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ${{ steps.package_name.outputs.name }} path: _package/${{ steps.package_name.outputs.name }} - name: Upload Environment - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ${{ matrix.name }}.env path: ${{ matrix.name }}.env @@ -189,7 +189,7 @@ jobs: shell: bash run: echo "UPLOAD_URL='${{ steps.create_release.outputs.upload_url }}'" > create-release.env - name: Upload create-release Environment - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: create-release.env path: create-release.env From 64fe3be77d6a64366573ef33b8157a44f7ebdbe9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Apr 2022 14:31:42 +0000 Subject: [PATCH 46/70] Bump actions/download-artifact from 2 to 3 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 2 to 3. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9f559c47..a36dd44f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -213,11 +213,11 @@ jobs: - ubuntu-18.04_x86_64 - ubuntu-20.04_x86_64 steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: ${{ matrix.name }}.env path: ${{ matrix.name }}.env - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: create-release.env path: create-release.env @@ -230,7 +230,7 @@ jobs: echo "::set-output name=content_type::$CONTENT_TYPE" echo "::set-output name=upload_url::$UPLOAD_URL" id: env - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: ${{ steps.env.outputs.package_name }} path: ${{ steps.env.outputs.package_name }} From 2218753d032646e20884a60ce5a02fda89e4819d Mon Sep 17 00:00:00 2001 From: melpon Date: Sat, 9 Apr 2022 03:58:25 +0900 Subject: [PATCH 47/70] =?UTF-8?q?CUDA=20=E3=81=8C=E5=AD=98=E5=9C=A8?= =?UTF-8?q?=E3=81=97=E3=81=AA=E3=81=84=E7=92=B0=E5=A2=83=E3=81=A7=E8=90=BD?= =?UTF-8?q?=E3=81=A1=E3=81=A6=E3=81=9F=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cuda/cuda_context_cuda.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cuda/cuda_context_cuda.cpp b/src/cuda/cuda_context_cuda.cpp index 75040eb9..6aece839 100644 --- a/src/cuda/cuda_context_cuda.cpp +++ b/src/cuda/cuda_context_cuda.cpp @@ -26,6 +26,10 @@ std::shared_ptr CudaContext::Create() { CUdevice device; CUcontext context; + if (!dyn::DynModule::Instance().IsLoadable(dyn::CUDA_SO)) { + throw std::exception(); + } + ckerror(dyn::cuInit(0)); ckerror(dyn::cuDeviceGet(&device, 0)); char device_name[80]; From d20f4896fc254448e35bc70df1c04356fb8e9a6f Mon Sep 17 00:00:00 2001 From: melpon Date: Mon, 11 Apr 2022 01:40:09 +0900 Subject: [PATCH 48/70] =?UTF-8?q?Intel=20Media=20SDK=20=E3=81=A7=E3=81=AE?= =?UTF-8?q?=E3=82=A8=E3=83=A9=E3=83=BC=E6=99=82=E3=81=AB=E3=83=AD=E3=82=B0?= =?UTF-8?q?=E3=82=92=E5=87=BA=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_msdk/msdk_session.cpp | 9 +++++++++ src/hwenc_msdk/msdk_video_decoder.cpp | 6 ++++++ src/hwenc_msdk/msdk_video_encoder.cpp | 24 +++++++++++++++--------- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/hwenc_msdk/msdk_session.cpp b/src/hwenc_msdk/msdk_session.cpp index e1d114c0..8457a8c8 100644 --- a/src/hwenc_msdk/msdk_session.cpp +++ b/src/hwenc_msdk/msdk_session.cpp @@ -1,5 +1,7 @@ #include "msdk_session.h" +#include + std::shared_ptr MsdkSession::Create() { std::shared_ptr session(new MsdkSession()); @@ -32,28 +34,33 @@ std::shared_ptr MsdkSession::Create() { sts = session->session.Init(impl, &ver); if (sts != MFX_ERR_NONE) { + std::cerr << "Failed to MFXInit: sts=" << sts << std::endl; return nullptr; } Microsoft::WRL::ComPtr idxgi_factory; if (FAILED(CreateDXGIFactory1(__uuidof(IDXGIFactory1), (void**)idxgi_factory.GetAddressOf()))) { + std::cerr << "Failed to CreateDXGIFactory1" << std::endl; return nullptr; } Microsoft::WRL::ComPtr idxgi_adapter; if (FAILED(idxgi_factory->EnumAdapters(0, idxgi_adapter.GetAddressOf()))) { + std::cerr << "Failed to EnumAdapters" << std::endl; return nullptr; } if (FAILED(D3D11CreateDevice(idxgi_adapter.Get(), D3D_DRIVER_TYPE_UNKNOWN, NULL, 0, NULL, 0, D3D11_SDK_VERSION, session->d3d11_device.GetAddressOf(), NULL, session->d3d11_context.GetAddressOf()))) { + std::cerr << "Failed to D3D11CreateDevice" << std::endl; return nullptr; } sts = session->session.SetHandle(MFX_HANDLE_D3D11_DEVICE, session->d3d11_device.Get()); if (sts != MFX_ERR_NONE) { + std::cerr << "Failed to MFXSetHandle: sts=" << sts << std::endl; return nullptr; } #endif @@ -61,11 +68,13 @@ std::shared_ptr MsdkSession::Create() { // Query selected implementation and version sts = session->session.QueryIMPL(&impl); if (sts != MFX_ERR_NONE) { + std::cerr << "Failed to MFXQueryIMPL: sts=" << sts << std::endl; return nullptr; } sts = session->session.QueryVersion(&ver); if (sts != MFX_ERR_NONE) { + std::cerr << "Failed to MFXQueryVersion: sts=" << sts << std::endl; return nullptr; } diff --git a/src/hwenc_msdk/msdk_video_decoder.cpp b/src/hwenc_msdk/msdk_video_decoder.cpp index abe02c51..3e90fe54 100644 --- a/src/hwenc_msdk/msdk_video_decoder.cpp +++ b/src/hwenc_msdk/msdk_video_decoder.cpp @@ -65,6 +65,12 @@ std::unique_ptr MsdkVideoDecoder::CreateDecoder( sts = decoder->Query(¶m, ¶m); if (sts == MFX_ERR_UNSUPPORTED) { + const char* codec_str = codec == MFX_CODEC_VP8 ? "MFX_CODEC_VP8" + : codec == MFX_CODEC_VP9 ? "MFX_CODEC_VP9" + : codec == MFX_CODEC_AV1 ? "MFX_CODEC_AV1" + : codec == MFX_CODEC_AVC ? "MFX_CODEC_AVC" + : "MFX_CODEC_UNKNOWN"; + std::cerr << "Unsupported decoder codec: codec=" << codec_str; return nullptr; } diff --git a/src/hwenc_msdk/msdk_video_encoder.cpp b/src/hwenc_msdk/msdk_video_encoder.cpp index cee3f05d..afe034d6 100644 --- a/src/hwenc_msdk/msdk_video_encoder.cpp +++ b/src/hwenc_msdk/msdk_video_encoder.cpp @@ -133,6 +133,12 @@ std::unique_ptr MsdkVideoEncoder::CreateEncoder( memcpy(&bk_param, ¶m, sizeof(bk_param)); sts = encoder->Query(¶m, ¶m); if (sts == MFX_ERR_UNSUPPORTED) { + const char* codec_str = codec == MFX_CODEC_VP8 ? "MFX_CODEC_VP8" + : codec == MFX_CODEC_VP9 ? "MFX_CODEC_VP9" + : codec == MFX_CODEC_AV1 ? "MFX_CODEC_AV1" + : codec == MFX_CODEC_AVC ? "MFX_CODEC_AVC" + : "MFX_CODEC_UNKNOWN"; + std::cerr << "Unsupported encoder codec: codec=" << codec_str; return nullptr; } } @@ -181,15 +187,15 @@ std::unique_ptr MsdkVideoEncoder::CreateEncoder( // F(IOPattern); //#undef F - //if (sts != MFX_ERR_NONE) { - // const char* codec_str = codec == MFX_CODEC_VP8 ? "MFX_CODEC_VP8" - // : codec == MFX_CODEC_VP9 ? "MFX_CODEC_VP9" - // : codec == MFX_CODEC_AV1 ? "MFX_CODEC_AV1" - // : codec == MFX_CODEC_AVC ? "MFX_CODEC_AVC" - // : "MFX_CODEC_UNKNOWN"; - // std::cerr << "Supported specified codec but has warning: codec=" - // << codec_str << " sts=" << sts << std::endl; - //} + if (sts != MFX_ERR_NONE) { + const char* codec_str = codec == MFX_CODEC_VP8 ? "MFX_CODEC_VP8" + : codec == MFX_CODEC_VP9 ? "MFX_CODEC_VP9" + : codec == MFX_CODEC_AV1 ? "MFX_CODEC_AV1" + : codec == MFX_CODEC_AVC ? "MFX_CODEC_AVC" + : "MFX_CODEC_UNKNOWN"; + std::cerr << "Supported specified codec but has warning: codec=" + << codec_str << " sts=" << sts << std::endl; + } if (init) { sts = encoder->Init(¶m); From e3b04e26eab8f1658b51659d4af437a8e68210f3 Mon Sep 17 00:00:00 2001 From: melpon Date: Mon, 11 Apr 2022 16:56:59 +0900 Subject: [PATCH 49/70] =?UTF-8?q?DX11=20=E3=83=87=E3=83=90=E3=82=A4?= =?UTF-8?q?=E3=82=B9=E3=82=92=20SDK=20=E3=81=AE=E3=82=B5=E3=83=B3=E3=83=97?= =?UTF-8?q?=E3=83=AB=E3=81=A8=E5=90=8C=E3=81=98=E4=BD=9C=E3=82=8A=E6=96=B9?= =?UTF-8?q?=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_msdk/msdk_session.cpp | 76 ++++++++++++++++++++++++++------- src/hwenc_msdk/msdk_session.h | 2 + 2 files changed, 63 insertions(+), 15 deletions(-) diff --git a/src/hwenc_msdk/msdk_session.cpp b/src/hwenc_msdk/msdk_session.cpp index 8457a8c8..df6a78e4 100644 --- a/src/hwenc_msdk/msdk_session.cpp +++ b/src/hwenc_msdk/msdk_session.cpp @@ -2,6 +2,55 @@ #include +MsdkSession::~MsdkSession() { + session.Close(); +} + +#ifdef _WIN32 + +#include + +const struct { + mfxIMPL impl; // actual implementation + mfxU32 adapterID; // device adapter number +} implTypes[] = {{MFX_IMPL_HARDWARE, 0}, + {MFX_IMPL_HARDWARE2, 1}, + {MFX_IMPL_HARDWARE3, 2}, + {MFX_IMPL_HARDWARE4, 3}}; + +static IDXGIAdapter* GetIntelDeviceAdapterHandle(mfxSession session) { + mfxU32 adapterNum = 0; + mfxIMPL impl; + + MFXQueryIMPL(session, &impl); + + mfxIMPL baseImpl = + MFX_IMPL_BASETYPE(impl); // Extract Media SDK base implementation type + + // get corresponding adapter number + for (mfxU8 i = 0; i < sizeof(implTypes) / sizeof(implTypes[0]); i++) { + if (implTypes[i].impl == baseImpl) { + adapterNum = implTypes[i].adapterID; + break; + } + } + + Microsoft::WRL::ComPtr factory; + HRESULT hres = CreateDXGIFactory(__uuidof(IDXGIFactory2), + (void**)factory.GetAddressOf()); + if (FAILED(hres)) + return NULL; + + IDXGIAdapter* adapter; + hres = factory->EnumAdapters(adapterNum, &adapter); + if (FAILED(hres)) + return NULL; + + return adapter; +} + +#endif + std::shared_ptr MsdkSession::Create() { std::shared_ptr session(new MsdkSession()); @@ -38,21 +87,18 @@ std::shared_ptr MsdkSession::Create() { return nullptr; } - Microsoft::WRL::ComPtr idxgi_factory; - if (FAILED(CreateDXGIFactory1(__uuidof(IDXGIFactory1), - (void**)idxgi_factory.GetAddressOf()))) { - std::cerr << "Failed to CreateDXGIFactory1" << std::endl; - return nullptr; - } - Microsoft::WRL::ComPtr idxgi_adapter; - if (FAILED(idxgi_factory->EnumAdapters(0, idxgi_adapter.GetAddressOf()))) { - std::cerr << "Failed to EnumAdapters" << std::endl; - return nullptr; - } - if (FAILED(D3D11CreateDevice(idxgi_adapter.Get(), D3D_DRIVER_TYPE_UNKNOWN, - NULL, 0, NULL, 0, D3D11_SDK_VERSION, - session->d3d11_device.GetAddressOf(), NULL, - session->d3d11_context.GetAddressOf()))) { + static D3D_FEATURE_LEVEL FeatureLevels[] = { + D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_10_1, + D3D_FEATURE_LEVEL_10_0}; + D3D_FEATURE_LEVEL pFeatureLevelsOut; + + Microsoft::WRL::ComPtr idxgi_adapter = + GetIntelDeviceAdapterHandle(session->session); + if (FAILED(D3D11CreateDevice( + idxgi_adapter.Get(), D3D_DRIVER_TYPE_UNKNOWN, NULL, 0, FeatureLevels, + (sizeof(FeatureLevels) / sizeof(FeatureLevels[0])), D3D11_SDK_VERSION, + session->d3d11_device.GetAddressOf(), &pFeatureLevelsOut, + session->d3d11_context.GetAddressOf()))) { std::cerr << "Failed to D3D11CreateDevice" << std::endl; return nullptr; } diff --git a/src/hwenc_msdk/msdk_session.h b/src/hwenc_msdk/msdk_session.h index 056705e9..1d9ac3ca 100644 --- a/src/hwenc_msdk/msdk_session.h +++ b/src/hwenc_msdk/msdk_session.h @@ -18,6 +18,8 @@ struct MsdkSession { MFXVideoSession session; + ~MsdkSession(); + #ifdef __linux__ std::unique_ptr libva; #endif From 74f43c2cc475debd8000b17060eb7f70438448c3 Mon Sep 17 00:00:00 2001 From: melpon Date: Mon, 11 Apr 2022 22:20:00 +0900 Subject: [PATCH 50/70] =?UTF-8?q?SetHandle=20=E3=82=92=E3=81=97=E3=81=AA?= =?UTF-8?q?=E3=81=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_msdk/msdk_session.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/hwenc_msdk/msdk_session.cpp b/src/hwenc_msdk/msdk_session.cpp index df6a78e4..868e7f6d 100644 --- a/src/hwenc_msdk/msdk_session.cpp +++ b/src/hwenc_msdk/msdk_session.cpp @@ -87,6 +87,7 @@ std::shared_ptr MsdkSession::Create() { return nullptr; } + /* static D3D_FEATURE_LEVEL FeatureLevels[] = { D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_10_1, D3D_FEATURE_LEVEL_10_0}; @@ -103,12 +104,19 @@ std::shared_ptr MsdkSession::Create() { return nullptr; } + Microsoft::WRL::ComPtr mt; + if (FAILED(session->d3d11_context->QueryInterface(mt.GetAddressOf()))) { + std::cerr << "Failed to QueryInterface" << std::endl; + } + mt->SetMultithreadProtected(true); + sts = session->session.SetHandle(MFX_HANDLE_D3D11_DEVICE, session->d3d11_device.Get()); if (sts != MFX_ERR_NONE) { std::cerr << "Failed to MFXSetHandle: sts=" << sts << std::endl; return nullptr; } + */ #endif // Query selected implementation and version From e8d03e8200798b8775849d0f6d08c1d335045325 Mon Sep 17 00:00:00 2001 From: melpon Date: Mon, 11 Apr 2022 23:24:46 +0900 Subject: [PATCH 51/70] =?UTF-8?q?Windows=20=E3=81=A7=20Intel=20Media=20SDK?= =?UTF-8?q?=20=E3=81=8C=E3=81=82=E3=82=8B=E3=81=8B=E3=81=A9=E3=81=86?= =?UTF-8?q?=E3=81=8B=E8=AA=BF=E3=81=B9=E3=81=A6=E3=81=AA=E3=81=8B=E3=81=A3?= =?UTF-8?q?=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/video_codec_info.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/video_codec_info.h b/src/video_codec_info.h index fd1b4ec2..08e8f747 100644 --- a/src/video_codec_info.h +++ b/src/video_codec_info.h @@ -121,6 +121,36 @@ struct VideoCodecInfo { } #endif +#if USE_MSDK_ENCODER + auto session = MsdkSession::Create(); + if (session != nullptr) { + if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_VP8)) { + info.vp8_encoders.push_back(Type::Intel); + } + if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_VP9)) { + info.vp9_encoders.push_back(Type::Intel); + } + if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_AVC)) { + info.h264_encoders.push_back(Type::Intel); + } + if (MsdkVideoEncoder::IsSupported(session, MFX_CODEC_AV1)) { + info.av1_encoders.push_back(Type::Intel); + } + if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_VP8)) { + info.vp8_decoders.push_back(Type::Intel); + } + if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_VP9)) { + info.vp9_decoders.push_back(Type::Intel); + } + if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_AVC)) { + info.h264_decoders.push_back(Type::Intel); + } + if (MsdkVideoDecoder::IsSupported(session, MFX_CODEC_AV1)) { + info.av1_decoders.push_back(Type::Intel); + } + } +#endif + info.vp8_encoders.push_back(Type::Software); info.vp8_decoders.push_back(Type::Software); info.vp9_encoders.push_back(Type::Software); From 632eb49ad21aee98cafb9fe53f9fe70f630a3dc9 Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 12 Apr 2022 00:31:19 +0900 Subject: [PATCH 52/70] =?UTF-8?q?=E7=84=A1=E4=BA=8B=E5=8B=95=E3=81=84?= =?UTF-8?q?=E3=81=9F=E3=81=AE=E3=81=A7=E3=82=B3=E3=83=BC=E3=83=89=E6=95=B4?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_msdk/msdk_session.cpp | 81 +-------------------------- src/hwenc_msdk/msdk_video_decoder.cpp | 12 ++-- src/hwenc_msdk/msdk_video_encoder.cpp | 23 ++++---- 3 files changed, 21 insertions(+), 95 deletions(-) diff --git a/src/hwenc_msdk/msdk_session.cpp b/src/hwenc_msdk/msdk_session.cpp index 868e7f6d..0098a77c 100644 --- a/src/hwenc_msdk/msdk_session.cpp +++ b/src/hwenc_msdk/msdk_session.cpp @@ -6,51 +6,6 @@ MsdkSession::~MsdkSession() { session.Close(); } -#ifdef _WIN32 - -#include - -const struct { - mfxIMPL impl; // actual implementation - mfxU32 adapterID; // device adapter number -} implTypes[] = {{MFX_IMPL_HARDWARE, 0}, - {MFX_IMPL_HARDWARE2, 1}, - {MFX_IMPL_HARDWARE3, 2}, - {MFX_IMPL_HARDWARE4, 3}}; - -static IDXGIAdapter* GetIntelDeviceAdapterHandle(mfxSession session) { - mfxU32 adapterNum = 0; - mfxIMPL impl; - - MFXQueryIMPL(session, &impl); - - mfxIMPL baseImpl = - MFX_IMPL_BASETYPE(impl); // Extract Media SDK base implementation type - - // get corresponding adapter number - for (mfxU8 i = 0; i < sizeof(implTypes) / sizeof(implTypes[0]); i++) { - if (implTypes[i].impl == baseImpl) { - adapterNum = implTypes[i].adapterID; - break; - } - } - - Microsoft::WRL::ComPtr factory; - HRESULT hres = CreateDXGIFactory(__uuidof(IDXGIFactory2), - (void**)factory.GetAddressOf()); - if (FAILED(hres)) - return NULL; - - IDXGIAdapter* adapter; - hres = factory->EnumAdapters(adapterNum, &adapter); - if (FAILED(hres)) - return NULL; - - return adapter; -} - -#endif - std::shared_ptr MsdkSession::Create() { std::shared_ptr session(new MsdkSession()); @@ -83,52 +38,22 @@ std::shared_ptr MsdkSession::Create() { sts = session->session.Init(impl, &ver); if (sts != MFX_ERR_NONE) { - std::cerr << "Failed to MFXInit: sts=" << sts << std::endl; - return nullptr; - } - - /* - static D3D_FEATURE_LEVEL FeatureLevels[] = { - D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_10_1, - D3D_FEATURE_LEVEL_10_0}; - D3D_FEATURE_LEVEL pFeatureLevelsOut; - - Microsoft::WRL::ComPtr idxgi_adapter = - GetIntelDeviceAdapterHandle(session->session); - if (FAILED(D3D11CreateDevice( - idxgi_adapter.Get(), D3D_DRIVER_TYPE_UNKNOWN, NULL, 0, FeatureLevels, - (sizeof(FeatureLevels) / sizeof(FeatureLevels[0])), D3D11_SDK_VERSION, - session->d3d11_device.GetAddressOf(), &pFeatureLevelsOut, - session->d3d11_context.GetAddressOf()))) { - std::cerr << "Failed to D3D11CreateDevice" << std::endl; + //std::cerr << "Failed to MFXInit: sts=" << sts << std::endl; return nullptr; } - Microsoft::WRL::ComPtr mt; - if (FAILED(session->d3d11_context->QueryInterface(mt.GetAddressOf()))) { - std::cerr << "Failed to QueryInterface" << std::endl; - } - mt->SetMultithreadProtected(true); - - sts = session->session.SetHandle(MFX_HANDLE_D3D11_DEVICE, - session->d3d11_device.Get()); - if (sts != MFX_ERR_NONE) { - std::cerr << "Failed to MFXSetHandle: sts=" << sts << std::endl; - return nullptr; - } - */ #endif // Query selected implementation and version sts = session->session.QueryIMPL(&impl); if (sts != MFX_ERR_NONE) { - std::cerr << "Failed to MFXQueryIMPL: sts=" << sts << std::endl; + //std::cerr << "Failed to MFXQueryIMPL: sts=" << sts << std::endl; return nullptr; } sts = session->session.QueryVersion(&ver); if (sts != MFX_ERR_NONE) { - std::cerr << "Failed to MFXQueryVersion: sts=" << sts << std::endl; + //std::cerr << "Failed to MFXQueryVersion: sts=" << sts << std::endl; return nullptr; } diff --git a/src/hwenc_msdk/msdk_video_decoder.cpp b/src/hwenc_msdk/msdk_video_decoder.cpp index 3e90fe54..2991ca0a 100644 --- a/src/hwenc_msdk/msdk_video_decoder.cpp +++ b/src/hwenc_msdk/msdk_video_decoder.cpp @@ -64,20 +64,20 @@ std::unique_ptr MsdkVideoDecoder::CreateDecoder( //qparam.NumExtParam = sizeof(ext_buffers) / sizeof(ext_buffers[0]); sts = decoder->Query(¶m, ¶m); - if (sts == MFX_ERR_UNSUPPORTED) { + if (sts < 0) { const char* codec_str = codec == MFX_CODEC_VP8 ? "MFX_CODEC_VP8" : codec == MFX_CODEC_VP9 ? "MFX_CODEC_VP9" : codec == MFX_CODEC_AV1 ? "MFX_CODEC_AV1" : codec == MFX_CODEC_AVC ? "MFX_CODEC_AVC" : "MFX_CODEC_UNKNOWN"; - std::cerr << "Unsupported decoder codec: codec=" << codec_str; + //std::cerr << "Unsupported decoder codec: codec=" << codec_str << std::endl; return nullptr; } - if (sts != MFX_ERR_NONE) { - std::cerr << "Supported specified codec but has warning: sts=" << sts - << std::endl; - } + //if (sts != MFX_ERR_NONE) { + // std::cout << "Supported specified codec but has warning: sts=" << sts + // << std::endl; + //} if (init) { // Initialize the Media SDK encoder diff --git a/src/hwenc_msdk/msdk_video_encoder.cpp b/src/hwenc_msdk/msdk_video_encoder.cpp index afe034d6..7dc6c028 100644 --- a/src/hwenc_msdk/msdk_video_encoder.cpp +++ b/src/hwenc_msdk/msdk_video_encoder.cpp @@ -132,13 +132,14 @@ std::unique_ptr MsdkVideoEncoder::CreateEncoder( } memcpy(&bk_param, ¶m, sizeof(bk_param)); sts = encoder->Query(¶m, ¶m); - if (sts == MFX_ERR_UNSUPPORTED) { + if (sts < 0) { const char* codec_str = codec == MFX_CODEC_VP8 ? "MFX_CODEC_VP8" : codec == MFX_CODEC_VP9 ? "MFX_CODEC_VP9" : codec == MFX_CODEC_AV1 ? "MFX_CODEC_AV1" : codec == MFX_CODEC_AVC ? "MFX_CODEC_AVC" : "MFX_CODEC_UNKNOWN"; - std::cerr << "Unsupported encoder codec: codec=" << codec_str; + //std::cerr << "Unsupported encoder codec: codec=" << codec_str + // << std::endl; return nullptr; } } @@ -187,15 +188,15 @@ std::unique_ptr MsdkVideoEncoder::CreateEncoder( // F(IOPattern); //#undef F - if (sts != MFX_ERR_NONE) { - const char* codec_str = codec == MFX_CODEC_VP8 ? "MFX_CODEC_VP8" - : codec == MFX_CODEC_VP9 ? "MFX_CODEC_VP9" - : codec == MFX_CODEC_AV1 ? "MFX_CODEC_AV1" - : codec == MFX_CODEC_AVC ? "MFX_CODEC_AVC" - : "MFX_CODEC_UNKNOWN"; - std::cerr << "Supported specified codec but has warning: codec=" - << codec_str << " sts=" << sts << std::endl; - } + //if (sts != MFX_ERR_NONE) { + // const char* codec_str = codec == MFX_CODEC_VP8 ? "MFX_CODEC_VP8" + // : codec == MFX_CODEC_VP9 ? "MFX_CODEC_VP9" + // : codec == MFX_CODEC_AV1 ? "MFX_CODEC_AV1" + // : codec == MFX_CODEC_AVC ? "MFX_CODEC_AVC" + // : "MFX_CODEC_UNKNOWN"; + // std::cerr << "Supported specified codec but has warning: codec=" + // << codec_str << " sts=" << sts << std::endl; + //} if (init) { sts = encoder->Init(¶m); From e5a1afc24f2a856f16975500b68f69d42ebc37a5 Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 12 Apr 2022 01:03:54 +0900 Subject: [PATCH 53/70] =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_msdk/msdk_video_encoder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hwenc_msdk/msdk_video_encoder.cpp b/src/hwenc_msdk/msdk_video_encoder.cpp index 7dc6c028..622c8cf4 100644 --- a/src/hwenc_msdk/msdk_video_encoder.cpp +++ b/src/hwenc_msdk/msdk_video_encoder.cpp @@ -118,7 +118,7 @@ std::unique_ptr MsdkVideoEncoder::CreateEncoder( mfxVideoParam bk_param; memcpy(&bk_param, ¶m, sizeof(bk_param)); sts = encoder->Query(¶m, ¶m); - if (sts == MFX_ERR_UNSUPPORTED) { + if (sts < 0) { memcpy(¶m, &bk_param, sizeof(bk_param)); // 失敗したら LowPower ON にした状態でもう一度確認する From 736e64abdb83c1fc5f8e3e54f01c34616567b9f6 Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 13 Apr 2022 15:25:46 +0900 Subject: [PATCH 54/70] =?UTF-8?q?=E3=83=87=E3=82=A4=E3=83=AA=E3=83=BC?= =?UTF-8?q?=E3=83=93=E3=83=AB=E3=83=89=E3=81=AB=E5=A4=B1=E6=95=97=E3=81=97?= =?UTF-8?q?=E3=81=A6=E3=82=8B=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/daily_build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/daily_build.yml b/.github/workflows/daily_build.yml index a2e8be5e..4a5b65df 100644 --- a/.github/workflows/daily_build.yml +++ b/.github/workflows/daily_build.yml @@ -17,6 +17,7 @@ jobs: runs-on: windows-2019 steps: - uses: actions/checkout@v3 + - uses: microsoft/setup-msbuild@v1.1 - name: Get Versions run: | Get-Content "VERSION" | Foreach-Object { From 8c9dccc2167da7e79e5033bcfcfb6964b2701f01 Mon Sep 17 00:00:00 2001 From: melpon Date: Sun, 17 Apr 2022 13:35:03 +0900 Subject: [PATCH 55/70] =?UTF-8?q?NOTICE=20=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- NOTICE | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/NOTICE b/NOTICE index dfaec0d4..68d2b4b9 100644 --- a/NOTICE +++ b/NOTICE @@ -1300,3 +1300,27 @@ consent, and any attempted assignment, subcontract, delegation, or transfer in violation of the foregoing will be null and void. The terms of this Agreement shall be binding upon assignees. ``` + +## Intel Media SDK + +``` +Copyright (c) 2017 Intel Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` From 58aad48796c23bdff811d33b3d53b61ee4969696 Mon Sep 17 00:00:00 2001 From: melpon Date: Sun, 17 Apr 2022 23:53:27 +0900 Subject: [PATCH 56/70] =?UTF-8?q?NvCodec=20=E3=81=8C=E6=9C=89=E5=8A=B9?= =?UTF-8?q?=E3=81=AA=E7=92=B0=E5=A2=83=E3=81=A7=20HW=20MJPEG=20=E3=83=87?= =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=80=E3=82=92=E4=BD=BF=E3=81=86=E5=A0=B4?= =?UTF-8?q?=E5=90=88=E3=80=81CUDA=20=E3=81=8C=E6=9C=89=E5=8A=B9=E3=81=A7?= =?UTF-8?q?=E3=81=82=E3=82=8B=E3=81=8B=E3=82=92=E8=AA=BF=E3=81=B9=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main.cpp b/src/main.cpp index dd3854fb..2ee89e31 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -102,6 +102,14 @@ int main(int argc, char* argv[]) { } #endif +#if USE_NVCODEC_ENCODER + // NvCodec が有効な環境で HW MJPEG デコーダを使う場合、CUDA が有効である必要がある + if (args.hw_mjpeg_decoder && cuda_context == nullptr) { + std::cerr << "Specified --hw-mjpeg-decoder=true but CUDA is invalid." + << std::endl; + } +#endif + auto capturer = ([&]() -> rtc::scoped_refptr { if (args.no_video_device) { return nullptr; From 88acac087def7663f2398b6596cef24bc7f99032 Mon Sep 17 00:00:00 2001 From: melpon Date: Mon, 18 Apr 2022 01:18:20 +0900 Subject: [PATCH 57/70] =?UTF-8?q?=E7=B5=82=E4=BA=86=E3=81=97=E5=BF=98?= =?UTF-8?q?=E3=82=8C=E3=81=A6=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.cpp b/src/main.cpp index 2ee89e31..06e22cac 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -107,6 +107,7 @@ int main(int argc, char* argv[]) { if (args.hw_mjpeg_decoder && cuda_context == nullptr) { std::cerr << "Specified --hw-mjpeg-decoder=true but CUDA is invalid." << std::endl; + return 2; } #endif From 48eafbd099ff3ccf193ae80d76a3d31ce96feef4 Mon Sep 17 00:00:00 2001 From: voluntas Date: Wed, 20 Apr 2022 11:25:41 +0900 Subject: [PATCH 58/70] Update USE_AYAME.md --- doc/USE_AYAME.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/USE_AYAME.md b/doc/USE_AYAME.md index 4e3d261d..366563c0 100644 --- a/doc/USE_AYAME.md +++ b/doc/USE_AYAME.md @@ -10,7 +10,7 @@ Ayame を利用してシグナリングサーバを立てるのが面倒な人 Ayame Labo は時雨堂が提供している Ayame を利用したサービスです。無料で利用可能です。 -https://ayame-labo.shiguredo.jp/ +https://ayame-labo.shiguredo.app/ ### Ayame Labo にサインアップしない場合 @@ -19,13 +19,13 @@ Ayame Labo はサインアップせずにシグナリングサーバを利用可 ここではルーム ID は `open-momo` としておりますが、必ず推測されにくい値に変更してください。 ```shell -./momo --no-audio-device ayame --signaling-url wss://ayame-labo.shiguredo.jp/signaling --channel-id open-momo +./momo --no-audio-device ayame --signaling-url wss://ayame-labo.shiguredo.app/signaling --channel-id open-momo ``` Windows の場合: ``` -.\momo.exe --no-audio-device ayame --signaling-url wss://ayame-labo.shiguredo.jp/signaling --channel-id open-momo +.\momo.exe --no-audio-device ayame --signaling-url wss://ayame-labo.shiguredo.app/signaling --channel-id open-momo ``` @@ -44,13 +44,13 @@ Ayame Labo にサインアップした場合はルーム ID に GitHub ユーザ - ここではシグナリングキーを `xyz` としています ```shell -./momo --no-audio-device ayame --signaling-url wss://ayame-labo.shiguredo.jp/signaling --channel-id shiguredo@open-momo --signaling-key xyz +./momo --no-audio-device ayame --signaling-url wss://ayame-labo.shiguredo.app/signaling --channel-id shiguredo@open-momo --signaling-key xyz ``` Windows の場合: ``` -.\momo.exe --no-audio-device ayame --signaling-url wss://ayame-labo.shiguredo.jp/signaling --channel-id shiguredo@open-momo --signaling-key xyz +.\momo.exe --no-audio-device ayame --signaling-url wss://ayame-labo.shiguredo.app/signaling --channel-id shiguredo@open-momo --signaling-key xyz ``` Ayame SDK のオンラインサンプルを利用します。 URL の引数にルーム ID とシグナリングキーを指定してアクセスします。 From 4a4b77fb6c8ef2fbda2fd2c1599d3a7f85a1bd23 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Wed, 20 Apr 2022 13:07:28 +0900 Subject: [PATCH 59/70] =?UTF-8?q?Boost=20=E3=81=AE=E3=83=90=E3=83=BC?= =?UTF-8?q?=E3=82=B8=E3=83=A7=E3=83=B3=E3=82=92=E4=B8=8A=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 27c305f9..e06fd8e2 100644 --- a/VERSION +++ b/VERSION @@ -1,10 +1,10 @@ MOMO_VERSION=2022.1.0 WEBRTC_BUILD_VERSION=100.4896.1.2 -BOOST_VERSION=1.78.0 +BOOST_VERSION=1.79.0 CLI11_VERSION=2.2.0 SDL2_VERSION=2.0.20 CMAKE_VERSION=3.22.3 CUDA_VERSION=11.0.2-1 WINCUDA_VERSION=10.2 LIBVA_VERSION=2.7.0 -MSDK_VERSION=20.1.1 \ No newline at end of file +MSDK_VERSION=20.1.1 From 547fc29cea46781d78bc15c30ef641f32c4626fc Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Sat, 23 Apr 2022 18:54:21 +0900 Subject: [PATCH 60/70] =?UTF-8?q?=E5=A4=89=E6=9B=B4=E5=B1=A5=E6=AD=B4?= =?UTF-8?q?=E3=82=92=E6=9B=B4=E6=96=B0=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 0bd8972d..7863bcd1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,8 @@ ## develop +- [UPDATE] Boost 1.79.0 に上げる + - @voluntas - [UPDATE] `libwebrtc` を `M100.4896@{#1}` に上げる - @tnoho - [FIX] libwebrtc m100 で make_ref_counted を使って scoped_refptr を作るようになったので修正 From 671f52aaf714392fd83b3b6cd69a7107c548dd32 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Sat, 23 Apr 2022 18:55:01 +0900 Subject: [PATCH 61/70] =?UTF-8?q?libwebrtc=20102=20=E3=81=AB=E4=B8=8A?= =?UTF-8?q?=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e06fd8e2..a354cba8 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ MOMO_VERSION=2022.1.0 -WEBRTC_BUILD_VERSION=100.4896.1.2 +WEBRTC_BUILD_VERSION=102.5005.1.0 BOOST_VERSION=1.79.0 CLI11_VERSION=2.2.0 SDL2_VERSION=2.0.20 From 8cef87882b574fa439bbc4562ec29946f389dfb2 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Sat, 23 Apr 2022 18:55:29 +0900 Subject: [PATCH 62/70] =?UTF-8?q?=E5=A4=89=E6=9B=B4=E5=B1=A5=E6=AD=B4?= =?UTF-8?q?=E3=82=92=E6=9B=B4=E6=96=B0=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7863bcd1..4ea40839 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,8 +13,8 @@ - [UPDATE] Boost 1.79.0 に上げる - @voluntas -- [UPDATE] `libwebrtc` を `M100.4896@{#1}` に上げる - - @tnoho +- [UPDATE] `libwebrtc` を `M102.5005@{#1}` に上げる + - @tnoho @voluntas - [FIX] libwebrtc m100 で make_ref_counted を使って scoped_refptr を作るようになったので修正 - @tnoho - [FIX] SDL のビルドが mac では declaration-after-statement に触れてビルドが通らないのでパッチで回避 From d5e3932814985521ab9fbda7cb8ac13545a05ff8 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Sat, 23 Apr 2022 18:56:40 +0900 Subject: [PATCH 63/70] =?UTF-8?q?=E3=83=90=E3=83=BC=E3=82=B8=E3=83=A7?= =?UTF-8?q?=E3=83=B3=E3=82=92=E4=B8=8A=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a354cba8..86228c58 100644 --- a/VERSION +++ b/VERSION @@ -3,7 +3,7 @@ WEBRTC_BUILD_VERSION=102.5005.1.0 BOOST_VERSION=1.79.0 CLI11_VERSION=2.2.0 SDL2_VERSION=2.0.20 -CMAKE_VERSION=3.22.3 +CMAKE_VERSION=3.23.1 CUDA_VERSION=11.0.2-1 WINCUDA_VERSION=10.2 LIBVA_VERSION=2.7.0 From 1614101ceafb502d9f5b22229d94997401eed9cb Mon Sep 17 00:00:00 2001 From: tnoho Date: Sat, 23 Apr 2022 22:10:35 +0900 Subject: [PATCH 64/70] =?UTF-8?q?typing=5Fdetection=20=E3=81=8C=E3=81=AA?= =?UTF-8?q?=E3=81=8F=E3=81=AA=E3=81=A3=E3=81=9F=E3=81=AE=E3=81=A7=E5=89=8A?= =?UTF-8?q?=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main.cpp | 1 - src/momo_args.h | 1 - src/rtc/rtc_manager.cpp | 2 -- src/rtc/rtc_manager.h | 1 - src/util.cpp | 2 -- 5 files changed, 7 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 06e22cac..f4f88335 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -196,7 +196,6 @@ int main(int argc, char* argv[]) { rtcm_config.disable_auto_gain_control = args.disable_auto_gain_control; rtcm_config.disable_noise_suppression = args.disable_noise_suppression; rtcm_config.disable_highpass_filter = args.disable_highpass_filter; - rtcm_config.disable_typing_detection = args.disable_typing_detection; rtcm_config.disable_residual_echo_detector = args.disable_residual_echo_detector; diff --git a/src/momo_args.h b/src/momo_args.h index 944e62fd..da6fe99e 100644 --- a/src/momo_args.h +++ b/src/momo_args.h @@ -78,7 +78,6 @@ struct MomoArgs { bool disable_auto_gain_control = false; bool disable_noise_suppression = false; bool disable_highpass_filter = false; - bool disable_typing_detection = false; bool disable_residual_echo_detector = false; VideoCodecInfo::Type vp8_encoder = VideoCodecInfo::Type::Default; diff --git a/src/rtc/rtc_manager.cpp b/src/rtc/rtc_manager.cpp index f110feb3..d78e1642 100644 --- a/src/rtc/rtc_manager.cpp +++ b/src/rtc/rtc_manager.cpp @@ -159,8 +159,6 @@ RTCManager::RTCManager( ao.noise_suppression = false; if (config_.disable_highpass_filter) ao.highpass_filter = false; - if (config_.disable_typing_detection) - ao.typing_detection = false; if (config_.disable_residual_echo_detector) ao.residual_echo_detector = false; RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << ao.ToString(); diff --git a/src/rtc/rtc_manager.h b/src/rtc/rtc_manager.h index 657c0c24..e941437c 100644 --- a/src/rtc/rtc_manager.h +++ b/src/rtc/rtc_manager.h @@ -34,7 +34,6 @@ struct RTCManagerConfig { bool disable_auto_gain_control = false; bool disable_noise_suppression = false; bool disable_highpass_filter = false; - bool disable_typing_detection = false; bool disable_residual_echo_detector = false; VideoCodecInfo::Type vp8_encoder = VideoCodecInfo::Type::Default; diff --git a/src/util.cpp b/src/util.cpp index 9ce242dc..335ebacb 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -189,8 +189,6 @@ void Util::ParseArgs(int argc, "Disable noise suppression for audio"); app.add_flag("--disable-highpass-filter", args.disable_highpass_filter, "Disable highpass filter for audio"); - app.add_flag("--disable-typing-detection", args.disable_typing_detection, - "Disable typing detection for audio"); app.add_flag("--disable-residual-echo-detector", args.disable_residual_echo_detector, "Disable residual echo detector for audio"); From 228469c986150e8f2b33d18f96f1823450afde9a Mon Sep 17 00:00:00 2001 From: melpon Date: Sun, 24 Apr 2022 02:13:32 +0900 Subject: [PATCH 65/70] =?UTF-8?q?msdk=E3=82=BB=E3=83=83=E3=82=B7=E3=83=A7?= =?UTF-8?q?=E3=83=B3=E3=82=92=E3=82=A8=E3=83=B3=E3=82=B3=E3=83=BC=E3=83=80?= =?UTF-8?q?/=E3=83=87=E3=82=B3=E3=83=BC=E3=83=80=E3=81=94=E3=81=A8?= =?UTF-8?q?=E3=81=AB=E7=94=9F=E6=88=90=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_msdk/msdk_session.h | 9 --------- src/main.cpp | 12 ------------ src/rtc/momo_video_decoder_factory.cpp | 8 ++++---- src/rtc/momo_video_decoder_factory.h | 3 --- src/rtc/momo_video_encoder_factory.cpp | 26 +++++++++++++------------- src/rtc/momo_video_encoder_factory.h | 3 --- src/rtc/rtc_manager.cpp | 6 ------ src/rtc/rtc_manager.h | 7 ------- 8 files changed, 17 insertions(+), 57 deletions(-) diff --git a/src/hwenc_msdk/msdk_session.h b/src/hwenc_msdk/msdk_session.h index 1d9ac3ca..02d9625f 100644 --- a/src/hwenc_msdk/msdk_session.h +++ b/src/hwenc_msdk/msdk_session.h @@ -10,11 +10,6 @@ #include "vaapi_utils_drm.h" #endif -#ifdef _WIN32 -#include -#include -#endif - struct MsdkSession { MFXVideoSession session; @@ -23,10 +18,6 @@ struct MsdkSession { #ifdef __linux__ std::unique_ptr libva; #endif -#ifdef _WIN32 - Microsoft::WRL::ComPtr d3d11_device; - Microsoft::WRL::ComPtr d3d11_context; -#endif static std::shared_ptr Create(); }; diff --git a/src/main.cpp b/src/main.cpp index f4f88335..68172c01 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -94,14 +94,6 @@ int main(int argc, char* argv[]) { } #endif -#if USE_MSDK_ENCODER - std::shared_ptr msdk_session; - try { - msdk_session = MsdkSession::Create(); - } catch (...) { - } -#endif - #if USE_NVCODEC_ENCODER // NvCodec が有効な環境で HW MJPEG デコーダを使う場合、CUDA が有効である必要がある if (args.hw_mjpeg_decoder && cuda_context == nullptr) { @@ -214,10 +206,6 @@ int main(int argc, char* argv[]) { rtcm_config.cuda_context = cuda_context; #endif -#if USE_MSDK_ENCODER - rtcm_config.msdk_session = msdk_session; -#endif - #if USE_SDL2 std::unique_ptr sdl_renderer = nullptr; if (args.use_sdl) { diff --git a/src/rtc/momo_video_decoder_factory.cpp b/src/rtc/momo_video_decoder_factory.cpp index 0be9abfc..e68d72ad 100644 --- a/src/rtc/momo_video_decoder_factory.cpp +++ b/src/rtc/momo_video_decoder_factory.cpp @@ -129,7 +129,7 @@ MomoVideoDecoderFactory::CreateVideoDecoder( #if USE_MSDK_ENCODER if (config_.vp8_decoder == VideoCodecInfo::Type::Intel) { return std::unique_ptr( - absl::make_unique(config_.msdk_session, + absl::make_unique(MsdkSession::Create(), MFX_CODEC_VP8)); } #endif @@ -156,7 +156,7 @@ MomoVideoDecoderFactory::CreateVideoDecoder( #if USE_MSDK_ENCODER if (config_.vp9_decoder == VideoCodecInfo::Type::Intel) { return std::unique_ptr( - absl::make_unique(config_.msdk_session, + absl::make_unique(MsdkSession::Create(), MFX_CODEC_VP9)); } #endif @@ -176,7 +176,7 @@ MomoVideoDecoderFactory::CreateVideoDecoder( #if USE_MSDK_ENCODER if (config_.av1_decoder == VideoCodecInfo::Type::Intel) { return std::unique_ptr( - absl::make_unique(config_.msdk_session, + absl::make_unique(MsdkSession::Create(), MFX_CODEC_AV1)); } #endif @@ -204,7 +204,7 @@ MomoVideoDecoderFactory::CreateVideoDecoder( #if USE_MSDK_ENCODER if (config_.h264_decoder == VideoCodecInfo::Type::Intel) { return std::unique_ptr( - absl::make_unique(config_.msdk_session, + absl::make_unique(MsdkSession::Create(), MFX_CODEC_AVC)); } #endif diff --git a/src/rtc/momo_video_decoder_factory.h b/src/rtc/momo_video_decoder_factory.h index 8480c569..a6534250 100644 --- a/src/rtc/momo_video_decoder_factory.h +++ b/src/rtc/momo_video_decoder_factory.h @@ -24,9 +24,6 @@ struct MomoVideoDecoderFactoryConfig { #if USE_NVCODEC_ENCODER std::shared_ptr cuda_context; #endif -#if USE_MSDK_ENCODER - std::shared_ptr msdk_session; -#endif }; class MomoVideoDecoderFactory : public webrtc::VideoDecoderFactory { diff --git a/src/rtc/momo_video_encoder_factory.cpp b/src/rtc/momo_video_encoder_factory.cpp index bcbf8a56..93187b34 100644 --- a/src/rtc/momo_video_encoder_factory.cpp +++ b/src/rtc/momo_video_encoder_factory.cpp @@ -108,7 +108,7 @@ MomoVideoEncoderFactory::GetSupportedFormats() const { } else if (config_.h264_encoder == VideoCodecInfo::Type::Intel) { #if USE_MSDK_ENCODER // Intel Media SDK の場合は対応してる場合のみ追加 - if (MsdkVideoEncoder::IsSupported(config_.msdk_session, MFX_CODEC_AVC)) { + if (MsdkVideoEncoder::IsSupported(MsdkSession::Create(), MFX_CODEC_AVC)) { for (const webrtc::SdpVideoFormat& format : h264_codecs) { supported_codecs.push_back(format); } @@ -178,10 +178,10 @@ MomoVideoEncoderFactory::CreateVideoEncoder( #endif #if USE_MSDK_ENCODER if (config_.vp8_encoder == VideoCodecInfo::Type::Intel) { - return WithSimulcast(format, [msdk_session = config_.msdk_session]( - const webrtc::SdpVideoFormat& format) { + return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return std::unique_ptr( - absl::make_unique(msdk_session, MFX_CODEC_VP8)); + absl::make_unique(MsdkSession::Create(), + MFX_CODEC_VP8)); }); } #endif @@ -203,10 +203,10 @@ MomoVideoEncoderFactory::CreateVideoEncoder( #endif #if USE_MSDK_ENCODER if (config_.vp9_encoder == VideoCodecInfo::Type::Intel) { - return WithSimulcast(format, [msdk_session = config_.msdk_session]( - const webrtc::SdpVideoFormat& format) { + return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return std::unique_ptr( - absl::make_unique(msdk_session, MFX_CODEC_VP9)); + absl::make_unique(MsdkSession::Create(), + MFX_CODEC_VP9)); }); } #endif @@ -222,10 +222,10 @@ MomoVideoEncoderFactory::CreateVideoEncoder( #endif #if USE_MSDK_ENCODER if (config_.av1_encoder == VideoCodecInfo::Type::Intel) { - return WithSimulcast(format, [msdk_session = config_.msdk_session]( - const webrtc::SdpVideoFormat& format) { + return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return std::unique_ptr( - absl::make_unique(msdk_session, MFX_CODEC_AV1)); + absl::make_unique(MsdkSession::Create(), + MFX_CODEC_AV1)); }); } #endif @@ -284,10 +284,10 @@ MomoVideoEncoderFactory::CreateVideoEncoder( #endif #if USE_MSDK_ENCODER if (config_.h264_encoder == VideoCodecInfo::Type::Intel) { - return WithSimulcast(format, [msdk_session = config_.msdk_session]( - const webrtc::SdpVideoFormat& format) { + return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return std::unique_ptr( - absl::make_unique(msdk_session, MFX_CODEC_AVC)); + absl::make_unique(MsdkSession::Create(), + MFX_CODEC_AVC)); }); } #endif diff --git a/src/rtc/momo_video_encoder_factory.h b/src/rtc/momo_video_encoder_factory.h index 45846b3a..3be280b1 100644 --- a/src/rtc/momo_video_encoder_factory.h +++ b/src/rtc/momo_video_encoder_factory.h @@ -29,9 +29,6 @@ struct MomoVideoEncoderFactoryConfig { #if defined(__linux__) && USE_NVCODEC_ENCODER std::shared_ptr cuda_context; #endif -#if USE_MSDK_ENCODER - std::shared_ptr msdk_session; -#endif }; class MomoVideoEncoderFactory : public webrtc::VideoEncoderFactory { diff --git a/src/rtc/rtc_manager.cpp b/src/rtc/rtc_manager.cpp index d78e1642..7d43b244 100644 --- a/src/rtc/rtc_manager.cpp +++ b/src/rtc/rtc_manager.cpp @@ -105,9 +105,6 @@ RTCManager::RTCManager( ec.hardware_encoder_only = cf.hardware_encoder_only; #if defined(__linux__) && USE_NVCODEC_ENCODER ec.cuda_context = cf.cuda_context; -#endif -#if USE_MSDK_ENCODER - ec.msdk_session = cf.msdk_session; #endif media_dependencies.video_encoder_factory = std::unique_ptr( @@ -119,9 +116,6 @@ RTCManager::RTCManager( dc.h264_decoder = resolve(cf.h264_decoder, info.h264_decoders); #if USE_NVCODEC_ENCODER dc.cuda_context = cf.cuda_context; -#endif -#if USE_MSDK_ENCODER - dc.msdk_session = cf.msdk_session; #endif media_dependencies.video_decoder_factory = std::unique_ptr( diff --git a/src/rtc/rtc_manager.h b/src/rtc/rtc_manager.h index e941437c..e46a86be 100644 --- a/src/rtc/rtc_manager.h +++ b/src/rtc/rtc_manager.h @@ -15,10 +15,6 @@ #include "video_codec_info.h" #include "video_track_receiver.h" -#if USE_MSDK_ENCODER -#include "hwenc_msdk/msdk_session.h" -#endif - struct RTCManagerConfig { bool insecure = false; @@ -59,9 +55,6 @@ struct RTCManagerConfig { #if USE_NVCODEC_ENCODER std::shared_ptr cuda_context; #endif -#if USE_MSDK_ENCODER - std::shared_ptr msdk_session; -#endif }; class RTCManager { From bb648bc35597721a258778d18910235a3c56cbb9 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Sun, 24 Apr 2022 17:52:06 +0900 Subject: [PATCH 66/70] =?UTF-8?q?=E3=83=90=E3=83=BC=E3=82=B8=E3=83=A7?= =?UTF-8?q?=E3=83=B3=E3=82=92=E4=B8=8A=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 86228c58..de3454ae 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ -MOMO_VERSION=2022.1.0 +MOMO_VERSION=2022.2.0 WEBRTC_BUILD_VERSION=102.5005.1.0 BOOST_VERSION=1.79.0 CLI11_VERSION=2.2.0 From 70b7e01b48389c3c7c3a005508e2709dab579a76 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Sun, 24 Apr 2022 17:53:10 +0900 Subject: [PATCH 67/70] =?UTF-8?q?=E5=A4=89=E6=9B=B4=E5=B1=A5=E6=AD=B4?= =?UTF-8?q?=E3=82=92=E6=9B=B4=E6=96=B0=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4ea40839..7537a0d8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,28 +11,30 @@ ## develop +## 2022.2.0 + +- [UPDATE] CLI11 を 2.2.0 に上げる + - @voluntas - [UPDATE] Boost 1.79.0 に上げる - @voluntas - [UPDATE] `libwebrtc` を `M102.5005@{#1}` に上げる - @tnoho @voluntas -- [FIX] libwebrtc m100 で make_ref_counted を使って scoped_refptr を作るようになったので修正 - - @tnoho -- [FIX] SDL のビルドが mac では declaration-after-statement に触れてビルドが通らないのでパッチで回避 - - @tnoho - [ADD] `--client-cert` と `--client-key` でクライアント認証をできるようにする - @melpon - [ADD] Windows と Ubuntu で NVIDIA VIDEO CODEC SDK を使ったハードウェアデコーダに対応 - @melpon -- [ADD] Windows と Ubuntu 20.04 で Intel Media SDK に対応 +- [ADD] Windows x86_64 と Ubuntu 20.04 x86_64 で Intel Media SDK に対応 - @melpon -- [UPDATE] CLI11 を 2.2.0 に上げる - - @voluntas - [FIX] Ubuntu 20.04 + H.264 + サイマルキャスト + --hw-mjpeg-decoder true で落ちるのを修正 (#221) - @melpon - [FIX] Raspberry Pi + H.264 + --hw-mjpeg-decoder true で、カメラの種類によっては動かないことがあるのを修正 (#141) - @melpon - [FIX] Raspberry Pi + H.264 + サイマルキャスト + --hw-mjpeg-decoder true で動かないのを修正 (#236) - @melpon +- [FIX] libwebrtc m100 で make_ref_counted を使って scoped_refptr を作るようになったので修正 + - @tnoho +- [FIX] SDL のビルドが mac では declaration-after-statement に触れてビルドが通らないのでパッチで回避 + - @tnoho ## 2022.1.0 From 873704fbb23873e70900f760a2cc9fbdf1b0ccd4 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Sun, 24 Apr 2022 18:09:13 +0900 Subject: [PATCH 68/70] =?UTF-8?q?REAMDE=20=E3=81=AE=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 380b7024..e0c85f4b 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,9 @@ https://momo.shiguredo.jp/ - [NVIDIA Jetson](https://www.nvidia.com/ja-jp/autonomous-machines/embedded-systems/) に搭載されている VP8 や VP9 や H.264 ハードウェアエンコーダー機能を利用することで 4K@30 での配信が可能です - [Raspberry Pi](https://www.raspberrypi.org/) の GPU に積まれている H.264 ハードウェアエンコーダー機能を利用することが可能です -- Apple macOS に搭載されている H.264 ハードウェアエンコーダー機能を [VideoToolbox](https://developer.apple.com/documentation/videotoolbox) 経由で利用することが可能です -- NVIDIA グラフィックスカードに搭載されている H.264 ハードウェアエンコーダー機能を [NVIDIA VIDEO CODEC SDK](https://developer.nvidia.com/nvidia-video-codec-sdk) 経由で利用することが可能です +- Apple macOS に搭載されている H.264 ハードウェアアクセラレーター機能を [VideoToolbox](https://developer.apple.com/documentation/videotoolbox) 経由で利用することが可能です +- NVIDIA グラフィックスカードに搭載されているハードウェアアクセラレーター機能を [NVIDIA VIDEO CODEC SDK](https://developer.nvidia.com/nvidia-video-codec-sdk) 経由で利用することが可能です +- [Intel Quick Sync Video](https://www.intel.co.jp/content/www/jp/ja/architecture-and-technology/quick-sync-video/quick-sync-video-general.html) を [Intel Media SDK](https://www.intel.com/content/www/us/en/developer/tools/media-sdk/overview.html) 経由で Windows x86_64 と Ubuntu x86_64 にて VP8 / VP9 / H.264 ハードウェアアクセラレーター機能を利用することが可能です ### 4K 30fps での配信 @@ -48,6 +49,9 @@ Momo を GUI 環境で利用した場合、[Simple DirectMedia Layer](https://ww AV1 の送受信に対応済みです。 +### クライアント証明書への対応 + +Momo は Sora モード利用時にクライアント証明書に対応しています。 ## 動画 [WebRTC Native Client Momo と Jetson Nano で 4K@30 配信](https://www.youtube.com/watch?v=z05bWtsgDPY) From 5a253e6e3faa408554d97142daa99b38c5d35e14 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Sun, 24 Apr 2022 18:10:33 +0900 Subject: [PATCH 69/70] =?UTF-8?q?=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e0c85f4b..d38cc6fc 100644 --- a/README.md +++ b/README.md @@ -189,16 +189,14 @@ limitations under the License. - 統計機能 - Ayame のシグナリング 経由での出力 - NVIDIA VIDEO CODEC SDK - - VP8 / VP9 / H.264 ハードウェアデコーダ対応 + - H.265 ハードウェアエンコーダ対応 - Ubuntu 20.04 - - Ubuntu 18.04 - Windows 10 - - MotionJPEG ハードウェアデコーダ対応 + - H.265 ハードウェアデコーダ対応 - Ubuntu 20.04 - - Ubuntu 18.04 - Windows 10 - Intel Media SDK 対応 - - VP8 / VP9 / H.264 / H.265 ハードウェアエンコーダ対応 + - H.265 ハードウェアエンコーダ対応 - 録画対応 - MP4 形式での出力 - WebM 形式での出力 From 421337955054bc1fc11d705592d42d01b1b413ea Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Sun, 24 Apr 2022 18:12:43 +0900 Subject: [PATCH 70/70] =?UTF-8?q?=E7=B5=B1=E4=B8=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index d38cc6fc..f860fa46 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ WebRTC Native Client Momo は libwebrtc を利用しブラウザなしで様々 https://momo.shiguredo.jp/ -### ハードウェアエンコーダへの対応 +### ハードウェアエンコーダーへの対応 - [NVIDIA Jetson](https://www.nvidia.com/ja-jp/autonomous-machines/embedded-systems/) に搭載されている VP8 や VP9 や H.264 ハードウェアエンコーダー機能を利用することで 4K@30 での配信が可能です - [Raspberry Pi](https://www.raspberrypi.org/) の GPU に積まれている H.264 ハードウェアエンコーダー機能を利用することが可能です @@ -31,7 +31,7 @@ https://momo.shiguredo.jp/ ### 4K 30fps での配信 -Momo はハードウェアエンコーダを利用することで WebRTC で 4K 60fps の配信を実現可能です +Momo はハードウェアエンコーダーを利用することで WebRTC で 4K 60fps の配信を実現可能です ### サイマルキャストへの対応 @@ -169,34 +169,34 @@ limitations under the License. **詳細は Discord やメールなどでお気軽にお問い合わせください** - Jetson AGX Orin 対応 - - AV1 ハードウェアエンコーダ対応 - - AV1 ハードウェアデコーダ対応 - - H.265 ハードウェアエンコーダ対応 - - H.265 ハードウェアデコーダ対応 + - AV1 ハードウェアエンコーダー対応 + - AV1 ハードウェアデコーダー対応 + - H.265 ハードウェアエンコーダー対応 + - H.265 ハードウェアデコーダー対応 - Jetson AGX Xavier 対応 - - H.265 ハードウェアエンコーダ対応 - - H.265 ハードウェアデコーダ対応 + - H.265 ハードウェアエンコーダー対応 + - H.265 ハードウェアデコーダー対応 - Jetson Xavier NX 対応 - - H.265 ハードウェアエンコーダ対応 - - H.265 ハードウェアデコーダ対応 + - H.265 ハードウェアエンコーダー対応 + - H.265 ハードウェアデコーダー対応 - Jetson Nano - - H.265 ハードウェアエンコーダ対応 - - H.265 ハードウェアデコーダ対応 + - H.265 ハードウェアエンコーダー対応 + - H.265 ハードウェアデコーダー対応 - Raspberry Pi 4 - - H.265 ハードウェアデコーダ対応 + - H.265 ハードウェアデコーダー対応 - AV1 対応 - Windows - 統計機能 - Ayame のシグナリング 経由での出力 - NVIDIA VIDEO CODEC SDK - - H.265 ハードウェアエンコーダ対応 + - H.265 ハードウェアエンコーダー対応 - Ubuntu 20.04 - Windows 10 - - H.265 ハードウェアデコーダ対応 + - H.265 ハードウェアデコーダー対応 - Ubuntu 20.04 - Windows 10 - Intel Media SDK 対応 - - H.265 ハードウェアエンコーダ対応 + - H.265 ハードウェアエンコーダー対応 - 録画対応 - MP4 形式での出力 - WebM 形式での出力 @@ -235,7 +235,7 @@ WebRTC Native Client に対する有料でのテクニカルサポート契約 ## H.264 のライセンス費用について -H.264 ハードウェアエンコーダ **のみ** を利用している Momo 単体の配布においてはライセンス費用は不要ですが、 +H.264 ハードウェアエンコーダー **のみ** を利用している Momo 単体の配布においてはライセンス費用は不要ですが、 ハードウェアとセットで配布する場合はライセンス費用を支払う必要があります。 ただし、 Raspberry Pi においては H.264 のライセンスがハードウェア費用に含まれているため、 @@ -243,15 +243,15 @@ H.264 ハードウェアエンコーダ **のみ** を利用している Momo 詳細については [MPEG LA](https://www.mpegla.com/) まで問い合わせる事をおすすめします。 -- Raspberry Pi のハードウェアエンコーダのライセンス費用は Raspberry Pi の価格に含まれています +- Raspberry Pi のハードウェアエンコーダーのライセンス費用は Raspberry Pi の価格に含まれています - https://www.raspberrypi.org/forums/viewtopic.php?t=200855 - Apple のライセンス費用は個人利用および非商用利用目的に限るため、配布においては別途、団体との契約が必要 - https://store.apple.com/Catalog/Japan/Images/EA0270_QTMPEG2.html -- AMD ビデオカードのハードウェアエンコーダのライセンス費用は別途、団体との契約が必要 +- AMD ビデオカードのハードウェアエンコーダーのライセンス費用は別途、団体との契約が必要 - https://github.com/GPUOpen-LibrariesAndSDKs/AMF/blob/master/amf/doc/AMF_API_Reference.pdf -- NVIDIA ビデオカードのハードウェアエンコーダのライセンス費用は別途、団体との契約が必要 +- NVIDIA ビデオカードのハードウェアエンコーダーのライセンス費用は別途、団体との契約が必要 - https://developer.download.nvidia.com/designworks/DesignWorks_SDKs_Samples_Tools_License_distrib_use_rights_2017_06_13.pdf -- NVIDIA Jetson Nano のハードウェアエンコーダのライセンス費用は別途、団体との契約が必要 - - [NVIDIA Jetson Nano 搭載の H\.264/H\.265 ハードウェアエンコーダのライセンスについて](https://medium.com/@voluntas/nvidia-jetson-nano-%E6%90%AD%E8%BC%89%E3%81%AE-h-264-h-265-%E3%83%8F%E3%83%BC%E3%83%89%E3%82%A6%E3%82%A7%E3%82%A2%E3%82%A8%E3%83%B3%E3%82%B3%E3%83%BC%E3%83%80%E3%81%AE%E3%83%A9%E3%82%A4%E3%82%BB%E3%83%B3%E3%82%B9%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6-ca207af302ee) -- Intel Quick Sync Video のハードウェアエンコーダライセンス費用は別途、団体との契約が必要 +- NVIDIA Jetson Nano のハードウェアエンコーダーのライセンス費用は別途、団体との契約が必要 + - [NVIDIA Jetson Nano 搭載の H\.264/H\.265 ハードウェアエンコーダーのライセンスについて](https://medium.com/@voluntas/nvidia-jetson-nano-%E6%90%AD%E8%BC%89%E3%81%AE-h-264-h-265-%E3%83%8F%E3%83%BC%E3%83%89%E3%82%A6%E3%82%A7%E3%82%A2%E3%82%A8%E3%83%B3%E3%82%B3%E3%83%BC%E3%83%80%E3%81%AE%E3%83%A9%E3%82%A4%E3%82%BB%E3%83%B3%E3%82%B9%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6-ca207af302ee) +- Intel Quick Sync Video のハードウェアエンコーダーライセンス費用は別途、団体との契約が必要 - [QuickSync \- H\.264 patent licensing fees \- Intel Community](https://community.intel.com/t5/Media-Intel-oneAPI-Video/QuickSync-H-264-patent-licensing-fees/td-p/921396)