From a28e1e4862b6686915596e77da669c8bbc338f89 Mon Sep 17 00:00:00 2001 From: Niyati Maheshwari Date: Wed, 1 May 2024 15:30:26 -0700 Subject: [PATCH] [Feature] Add H265 support with SDP, RTP payloader-deplayloader (#1965) * Add RtpH265Payloader.c and RtpH265Payloader.h * Add support for H265 in PeerConnection/SessionDescription * Add support for H265 in PeerConnection/PeerConnection * Add support for H265 in PeerConnection/Rtp * Add support for H265 in samples/Common.c * Add support for H265 in samples/kvsWebRTCClientMaster.c * rtp, sdp fix, flag removed, clang fixed, windows build fixed, new test added * test fix * cleanup * cleanup * remove #if 0 * clang * presentation ts fix * clang-format fix * PKG_CONFIG_PATH in kvscommon * missing bracket * fix all builds * ci * cleanup * fix windows build, rename h265 defs * remove duplicate line from h264 and h265 * sample changes * address comments * clang-format * gst sample * cleanup args * clang-format * cleanup * add sdp tests * address comments * address commentas * set default payload type only once * address comments * fix height and width * sdp change --------- Co-authored-by: Hongli Wang --- .github/workflows/ci.yml | 2 +- README.md | 29 +- samples/Common.c | 9 +- samples/GstAudioVideoReceiver.c | 25 +- samples/Samples.h | 11 + samples/kvsWebRTCClientMaster.c | 51 +++- samples/kvsWebRTCClientMasterGstSample.c | 93 ++++-- samples/kvsWebRTCClientViewer.c | 26 ++ samples/kvsWebRTCClientViewerGstSample.c | 62 ++-- src/source/Include_i.h | 1 + src/source/PeerConnection/PeerConnection.c | 21 +- src/source/PeerConnection/PeerConnection.h | 1 + src/source/PeerConnection/Rtp.c | 5 + .../PeerConnection/SessionDescription.c | 71 ++++- .../PeerConnection/SessionDescription.h | 7 +- src/source/Rtp/Codecs/RtpH264Payloader.c | 8 +- src/source/Rtp/Codecs/RtpH265Payloader.c | 283 ++++++++++++++++++ src/source/Rtp/Codecs/RtpH265Payloader.h | 41 +++ tst/RtpFunctionalityTest.cpp | 115 ++++++- tst/SdpApiTest.cpp | 61 ++++ tst/WebRTCClientTestFixture.h | 13 +- 21 files changed, 856 insertions(+), 79 deletions(-) create mode 100644 src/source/Rtp/Codecs/RtpH265Payloader.c create mode 100644 src/source/Rtp/Codecs/RtpH265Payloader.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e7ef065a9..f183f6fd2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -645,7 +645,7 @@ jobs: shell: powershell run: | $env:Path += ';C:\webrtc\open-source\bin;C:\tools\pthreads-w32-2-9-1-release\Pre-built.2\dll\x64;C:\webrtc\build' - & "C:\webrtc\build\tst\webrtc_client_test.exe" --gtest_filter="-SignalingApiFunctionalityTest.receivingIceConfigOffer_SlowClockSkew:SignalingApiFunctionalityTest.iceServerConfigRefreshConnectedAuthExpiration:SignalingApiFunctionalityTest.receivingIceConfigOffer_FastClockSkew:SignalingApiFunctionalityTest.receivingIceConfigOffer_FastClockSkew_VerifyOffsetRemovedWhenClockFixed:DataChannelFunctionalityTest.*:DtlsApiTest.*:IceApiTest.*:IceFunctionalityTest.*:PeerConnectionFunctionalityTest.*:TurnConnectionFunctionalityTest.*:RtpFunctionalityTest.marshallUnmarshallH264Data:RtpFunctionalityTest.packingUnpackingVerifySameH264Frame:RtcpFunctionalityTest.onRtcpPacketCompound:RtcpFunctionalityTest.twcc3" + & "C:\webrtc\build\tst\webrtc_client_test.exe" --gtest_filter="-SignalingApiFunctionalityTest.receivingIceConfigOffer_SlowClockSkew:SignalingApiFunctionalityTest.iceServerConfigRefreshConnectedAuthExpiration:SignalingApiFunctionalityTest.receivingIceConfigOffer_FastClockSkew:SignalingApiFunctionalityTest.receivingIceConfigOffer_FastClockSkew_VerifyOffsetRemovedWhenClockFixed:DataChannelFunctionalityTest.*:DtlsApiTest.*:IceApiTest.*:IceFunctionalityTest.*:PeerConnectionFunctionalityTest.*:TurnConnectionFunctionalityTest.*:RtpFunctionalityTest.marshallUnmarshallH264Data:RtpFunctionalityTest.packingUnpackingVerifySameH264Frame:RtpFunctionalityTest.packingUnpackingVerifySameH265Frame:RtcpFunctionalityTest.onRtcpPacketCompound:RtcpFunctionalityTest.twcc3" # windows-msvc-mbedtls: # runs-on: windows-2022 # env: diff --git a/README.md b/README.md index 586c8c0602..114891e447 100644 --- a/README.md +++ b/README.md @@ -273,15 +273,18 @@ After executing `make` you will have sample applications in your `build/samples` #### Sample: kvsWebrtcClientMaster This application sends sample H264/Opus frames (path: `/samples/h264SampleFrames` and `/samples/opusSampleFrames`) via WebRTC. It also accepts incoming audio, if enabled in the browser. When checked in the browser, it prints the metadata of the received audio packets in your terminal. To run: ```shell -./samples/kvsWebrtcClientMaster +./samples/kvsWebrtcClientMaster ``` To use the **Storage for WebRTC** feature, run the same command as above but with an additional command line arg to enable the feature. ```shell -./samples/kvsWebrtcClientMaster 1 +./samples/kvsWebrtcClientMaster 1 ``` +Allowed audio-codec: opus (default codec if nothing is specified), aac +Allowed video-codec: h264 (default codec if nothing is specified), h265 + #### Sample: kvsWebrtcClientMasterGstSample This application can send media from a GStreamer pipeline using test H264/Opus frames, device `autovideosrc` and `autoaudiosrc` input, or a received RTSP stream. It also will playback incoming audio via an `autoaudiosink`. To run: ```shell @@ -292,23 +295,41 @@ Pass the desired media and source type when running the sample. The mediaType ca ./samples/kvsWebrtcClientMasterGstSample rtspsrc rtsp:// ``` +Using the testsrc with audio and video-codec +```shell +./samples/kvsWebrtcClientMasterGstSample +``` + +Example: +```shell +./samples/kvsWebrtcClientMasterGstSample audio-video testsrc opus h264 +``` + +Allowed audio-codec: opus (default codec if nothing is specified), aac +Allowed video-codec: h264 (default codec if nothing is specified), h265 #### Sample: kvsWebrtcClientViewer This application accepts sample H264/Opus frames by default. You can use other supported codecs by changing the value for `videoTrack.codec` and `audioTrack.codec` in _Common.c_. By default, this sample only logs the size of the audio and video buffer it receives. To write these frames to a file using GStreamer, use the _kvsWebrtcClientViewerGstSample_ instead. To run: ```shell -./samples/kvsWebrtcClientViewer +./samples/kvsWebrtcClientViewer ``` +Allowed audio-codec: opus (default codec if nothing is specified), aac +Allowed video-codec: h264 (default codec if nothing is specified), h265 + #### Sample: kvsWebrtcClientViewerGstSample This application is similar to the kvsWebrtcClientViewer. However, instead of just logging the media it receives, it generates a file using filesink. Make sure that your device has enough space to write the media to a file. You can also customize the receiving logic by modifying the functions in _GstAudioVideoReceiver.c_ To run: ```shell -./samples/kvsWebrtcClientViewerGstSample +./samples/kvsWebrtcClientViewerGstSample ``` +Allowed audio-codec: opus (default codec if nothing is specified), aac +Allowed video-codec: h264 (default codec if nothing is specified), h265 + ##### Known issues: Our GStreamer samples leverage [MatroskaMux](https://gstreamer.freedesktop.org/documentation/matroska/matroskamux.html?gi-language=c) to receive media from its peer and save it to a file. However, MatroskaMux is designed for scenarios where the media's format remains constant throughout streaming. When the media's format changes mid-streaming (referred to as "caps changes"), MatroskaMux encounters limitations, its behavior cannot be predicted and it may be unable to handle these changes, resulting in an error message like: diff --git a/samples/Common.c b/samples/Common.c index 0991cc03f0..d512c4c559 100644 --- a/samples/Common.c +++ b/samples/Common.c @@ -571,13 +571,12 @@ STATUS createSampleStreamingSession(PSampleConfiguration pSampleConfiguration, P } #endif - // Declare that we support H264,Profile=42E01F,level-asymmetry-allowed=1,packetization-mode=1 and Opus - CHK_STATUS(addSupportedCodec(pSampleStreamingSession->pPeerConnection, RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE)); - CHK_STATUS(addSupportedCodec(pSampleStreamingSession->pPeerConnection, RTC_CODEC_OPUS)); + CHK_STATUS(addSupportedCodec(pSampleStreamingSession->pPeerConnection, pSampleConfiguration->videoCodec)); + CHK_STATUS(addSupportedCodec(pSampleStreamingSession->pPeerConnection, pSampleConfiguration->audioCodec)); // Add a SendRecv Transceiver of type video videoTrack.kind = MEDIA_STREAM_TRACK_KIND_VIDEO; - videoTrack.codec = RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE; + videoTrack.codec = pSampleConfiguration->videoCodec; videoRtpTransceiverInit.direction = RTC_RTP_TRANSCEIVER_DIRECTION_SENDRECV; videoRtpTransceiverInit.rollingBufferDurationSec = 3; // Considering 4 Mbps for 720p (which is what our samples use). This is for H.264. @@ -593,7 +592,7 @@ STATUS createSampleStreamingSession(PSampleConfiguration pSampleConfiguration, P // Add a SendRecv Transceiver of type audio audioTrack.kind = MEDIA_STREAM_TRACK_KIND_AUDIO; - audioTrack.codec = RTC_CODEC_OPUS; + audioTrack.codec = pSampleConfiguration->audioCodec; audioRtpTransceiverInit.direction = RTC_RTP_TRANSCEIVER_DIRECTION_SENDRECV; audioRtpTransceiverInit.rollingBufferDurationSec = 3; // For opus, the bitrate could be between 6 Kbps to 510 Kbps diff --git a/samples/GstAudioVideoReceiver.c b/samples/GstAudioVideoReceiver.c index c961d5895f..626a963eb3 100644 --- a/samples/GstAudioVideoReceiver.c +++ b/samples/GstAudioVideoReceiver.c @@ -5,6 +5,7 @@ static UINT64 presentationTsIncrement = 0; static BOOL eos = FALSE; +static RTC_CODEC audioCodec = RTC_CODEC_OPUS; // This function is a callback for the transceiver for every single video frame it receives // It writes these frames to a buffer and pushes it to the `appsrcVideo` element of the @@ -67,7 +68,13 @@ VOID onGstAudioFrameReady(UINT64 customData, PFrame pFrame) GST_BUFFER_DTS(buffer) = presentationTsIncrement; GST_BUFFER_PTS(buffer) = presentationTsIncrement; - GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale(pFrame->size, GST_SECOND, DEFAULT_AUDIO_OPUS_BYTE_RATE); + + if (audioCodec == RTC_CODEC_AAC) { + GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale(pFrame->size, GST_SECOND, DEFAULT_AUDIO_AAC_BYTE_RATE); + } else { + GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale(pFrame->size, GST_SECOND, DEFAULT_AUDIO_OPUS_BYTE_RATE); + } + // TODO: check for other codecs once the pipelines are added if (gst_buffer_fill(buffer, 0, pFrame->frameData, pFrame->size) != pFrame->size) { DLOGE("Buffer fill did not complete correctly"); @@ -111,9 +118,11 @@ PVOID receiveGstreamerAudioVideo(PVOID args) roleType = "Master"; } - CHK_ERR(gst_init_check(NULL, NULL, &error), STATUS_INTERNAL_ERROR, "[KVS %s] GStreamer initialization failed"); + audioCodec = pSampleConfiguration->audioCodec; + + CHK_ERR(gst_init_check(NULL, NULL, &error), STATUS_INTERNAL_ERROR, "[KVS GStreamer %s] GStreamer initialization failed"); - CHK_ERR(pSampleStreamingSession != NULL, STATUS_NULL_ARG, "[KVS %s] Sample streaming session is NULL", roleType); + CHK_ERR(pSampleStreamingSession != NULL, STATUS_NULL_ARG, "[KVS Gstreamer %s] Sample streaming session is NULL", roleType); // It is advised to modify the pipeline and the caps as per the source of the media. Customers can also modify this pipeline to // use any other sinks instead of `filesink` like `autovideosink` and `autoaudiosink`. The existing pipelines are not complex enough to @@ -168,17 +177,17 @@ PVOID receiveGstreamerAudioVideo(PVOID args) audioVideoDescription = g_strjoin(" ", videoDescription, audioDescription, NULL); pipeline = gst_parse_launch(audioVideoDescription, &error); - CHK_ERR(pipeline != NULL, STATUS_INTERNAL_ERROR, "[KVS %s] Pipeline is NULL", roleType); + CHK_ERR(pipeline != NULL, STATUS_INTERNAL_ERROR, "[KVS GStreamer %s] Pipeline is NULL", roleType); appsrcVideo = gst_bin_get_by_name(GST_BIN(pipeline), "appsrc-video"); - CHK_ERR(appsrcVideo != NULL, STATUS_INTERNAL_ERROR, "[KVS %s] Cannot find appsrc video", roleType); + CHK_ERR(appsrcVideo != NULL, STATUS_INTERNAL_ERROR, "[KVS GStreamer %s] Cannot find appsrc video", roleType); CHK_STATUS(transceiverOnFrame(pSampleStreamingSession->pVideoRtcRtpTransceiver, (UINT64) appsrcVideo, onGstVideoFrameReady)); g_object_set(G_OBJECT(appsrcVideo), "caps", videocaps, NULL); gst_caps_unref(videocaps); if (pSampleConfiguration->mediaType == SAMPLE_STREAMING_AUDIO_VIDEO) { appsrcAudio = gst_bin_get_by_name(GST_BIN(pipeline), "appsrc-audio"); - CHK_ERR(appsrcAudio != NULL, STATUS_INTERNAL_ERROR, "[KVS %s] Cannot find appsrc audio", roleType); + CHK_ERR(appsrcAudio != NULL, STATUS_INTERNAL_ERROR, "[KVS GStreamer %s] Cannot find appsrc audio", roleType); CHK_STATUS(transceiverOnFrame(pSampleStreamingSession->pAudioRtcRtpTransceiver, (UINT64) appsrcAudio, onGstAudioFrameReady)); g_object_set(G_OBJECT(appsrcAudio), "caps", audiocaps, NULL); gst_caps_unref(audiocaps); @@ -191,7 +200,7 @@ PVOID receiveGstreamerAudioVideo(PVOID args) /* block until error or EOS */ bus = gst_element_get_bus(pipeline); - CHK_ERR(bus != NULL, STATUS_INTERNAL_ERROR, "[KVS %s] Bus is NULL", roleType); + CHK_ERR(bus != NULL, STATUS_INTERNAL_ERROR, "[KVS GStreamer %s] Bus is NULL", roleType); msg = gst_bus_timed_pop_filtered(bus, GST_CLOCK_TIME_NONE, GST_MESSAGE_ERROR | GST_MESSAGE_EOS); /* Free resources */ @@ -226,7 +235,7 @@ PVOID receiveGstreamerAudioVideo(PVOID args) CleanUp: if (error != NULL) { - DLOGE("[KVS %s] %s", roleType, error->message); + DLOGE("[KVS GStreamer %s] %s", roleType, error->message); g_clear_error(&error); } diff --git a/samples/Samples.h b/samples/Samples.h index 9e3dfa5bd9..e409a9b5dd 100644 --- a/samples/Samples.h +++ b/samples/Samples.h @@ -13,6 +13,7 @@ extern "C" { #include #define NUMBER_OF_H264_FRAME_FILES 1500 +#define NUMBER_OF_H265_FRAME_FILES 1500 #define NUMBER_OF_OPUS_FRAME_FILES 618 #define DEFAULT_FPS_VALUE 25 #define DEFAULT_VIDEO_HEIGHT_PIXELS 720 @@ -25,6 +26,14 @@ extern "C" { #define DEFAULT_AUDIO_AAC_BITS_PER_SAMPLE 16 #define DEFAULT_MAX_CONCURRENT_STREAMING_SESSION 10 +#define AUDIO_CODEC_NAME_ALAW "alaw" +#define AUDIO_CODEC_NAME_MULAW "mulaw" +#define AUDIO_CODEC_NAME_OPUS "opus" +#define AUDIO_CODEC_NAME_AAC "aac" +#define VIDEO_CODEC_NAME_H264 "h264" +#define VIDEO_CODEC_NAME_H265 "h265" +#define VIDEO_CODEC_NAME_VP8 "vp8" + #define SAMPLE_MASTER_CLIENT_ID "ProducerMaster" #define SAMPLE_VIEWER_CLIENT_ID "ConsumerViewer" #define SAMPLE_CHANNEL_NAME (PCHAR) "ScaryTestChannel" @@ -123,6 +132,8 @@ typedef struct { PCHAR pCaCertPath; PAwsCredentialProvider pCredentialProvider; SIGNALING_CLIENT_HANDLE signalingClientHandle; + RTC_CODEC audioCodec; + RTC_CODEC videoCodec; PBYTE pAudioFrameBuffer; UINT32 audioBufferSize; PBYTE pVideoFrameBuffer; diff --git a/samples/kvsWebRTCClientMaster.c b/samples/kvsWebRTCClientMaster.c index d3955307a5..17411238a0 100644 --- a/samples/kvsWebRTCClientMaster.c +++ b/samples/kvsWebRTCClientMaster.c @@ -10,6 +10,8 @@ INT32 main(INT32 argc, CHAR* argv[]) PCHAR pChannelName; SignalingClientMetrics signalingClientMetrics; signalingClientMetrics.version = SIGNALING_CLIENT_METRICS_CURRENT_VERSION; + RTC_CODEC audioCodec = RTC_CODEC_OPUS; + RTC_CODEC videoCodec = RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE; SET_INSTRUMENTED_ALLOCATORS(); UINT32 logLevel = setLogLevel(); @@ -27,10 +29,28 @@ INT32 main(INT32 argc, CHAR* argv[]) CHK_STATUS(createSampleConfiguration(pChannelName, SIGNALING_CHANNEL_ROLE_TYPE_MASTER, TRUE, TRUE, logLevel, &pSampleConfiguration)); + if (argc > 3) { + if (!STRCMP(argv[3], AUDIO_CODEC_NAME_AAC)) { + audioCodec = RTC_CODEC_AAC; + } else { + DLOGI("[KVS Master] Defaulting to opus as the specified codec's sample frames may not be available"); + } + } + + if (argc > 4) { + if (!STRCMP(argv[4], VIDEO_CODEC_NAME_H265)) { + videoCodec = RTC_CODEC_H265; + } else { + DLOGI("[KVS Master] Defaulting to H264 as the specified codec's sample frames may not be available"); + } + } + // Set the audio and video handlers pSampleConfiguration->audioSource = sendAudioPackets; pSampleConfiguration->videoSource = sendVideoPackets; pSampleConfiguration->receiveAudioVideoSource = sampleReceiveAudioVideoFrame; + pSampleConfiguration->audioCodec = audioCodec; + pSampleConfiguration->videoCodec = videoCodec; if (argc > 2 && STRNCMP(argv[2], "1", 2) == 0) { pSampleConfiguration->channelInfo.useMediaStorage = TRUE; @@ -44,11 +64,21 @@ INT32 main(INT32 argc, CHAR* argv[]) // Check if the samples are present - CHK_STATUS(readFrameFromDisk(NULL, &frameSize, "./h264SampleFrames/frame-0001.h264")); - DLOGI("[KVS Master] Checked sample video frame availability....available"); + if (videoCodec == RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE) { + CHK_STATUS(readFrameFromDisk(NULL, &frameSize, "./h264SampleFrames/frame-0001.h264")); + DLOGI("[KVS Master] Checked H264 sample video frame availability....available"); + } else if (videoCodec == RTC_CODEC_H265) { + CHK_STATUS(readFrameFromDisk(NULL, &frameSize, "./h265SampleFrames/frame-0001.h265")); + DLOGI("[KVS Master] Checked H265 sample video frame availability....available"); + } - CHK_STATUS(readFrameFromDisk(NULL, &frameSize, "./opusSampleFrames/sample-001.opus")); - DLOGI("[KVS Master] Checked sample audio frame availability....available"); + if (audioCodec == RTC_CODEC_OPUS) { + CHK_STATUS(readFrameFromDisk(NULL, &frameSize, "./opusSampleFrames/sample-001.opus")); + DLOGI("[KVS Master] Checked Opus sample audio frame availability....available"); + } else if (audioCodec == RTC_CODEC_AAC) { + CHK_STATUS(readFrameFromDisk(NULL, &frameSize, "./aacSampleFrames/sample-001.aac")); + DLOGI("[KVS Master] Checked AAC sample audio frame availability....available"); + } // Initialize KVS WebRTC. This must be done before anything else, and must only be done once. CHK_STATUS(initKvsWebRtc()); @@ -146,7 +176,11 @@ PVOID sendVideoPackets(PVOID args) while (!ATOMIC_LOAD_BOOL(&pSampleConfiguration->appTerminateFlag)) { fileIndex = fileIndex % NUMBER_OF_H264_FRAME_FILES + 1; - SNPRINTF(filePath, MAX_PATH_LEN, "./h264SampleFrames/frame-%04d.h264", fileIndex); + if (pSampleConfiguration->videoCodec == RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE) { + SNPRINTF(filePath, MAX_PATH_LEN, "./h264SampleFrames/frame-%04d.h264", fileIndex); + } else if (pSampleConfiguration->videoCodec == RTC_CODEC_H265) { + SNPRINTF(filePath, MAX_PATH_LEN, "./h265SampleFrames/frame-%04d.h265", fileIndex); + } CHK_STATUS(readFrameFromDisk(NULL, &frameSize, filePath)); @@ -218,7 +252,12 @@ PVOID sendAudioPackets(PVOID args) while (!ATOMIC_LOAD_BOOL(&pSampleConfiguration->appTerminateFlag)) { fileIndex = fileIndex % NUMBER_OF_OPUS_FRAME_FILES + 1; - SNPRINTF(filePath, MAX_PATH_LEN, "./opusSampleFrames/sample-%03d.opus", fileIndex); + + if (pSampleConfiguration->audioCodec == RTC_CODEC_AAC) { + SNPRINTF(filePath, MAX_PATH_LEN, "./aacSampleFrames/sample-%03d.aac", fileIndex); + } else if (pSampleConfiguration->audioCodec == RTC_CODEC_OPUS) { + SNPRINTF(filePath, MAX_PATH_LEN, "./opusSampleFrames/sample-%03d.opus", fileIndex); + } CHK_STATUS(readFrameFromDisk(NULL, &frameSize, filePath)); diff --git a/samples/kvsWebRTCClientMasterGstSample.c b/samples/kvsWebRTCClientMasterGstSample.c index ff5ffccb35..6b60070d9f 100644 --- a/samples/kvsWebRTCClientMasterGstSample.c +++ b/samples/kvsWebRTCClientMasterGstSample.c @@ -174,7 +174,7 @@ PVOID sendGstreamerAudioVideo(PVOID args) * For H265/AAC * pipeline = * gst_parse_launch("videotestsrc pattern=ball is-live=TRUE ! timeoverlay ! queue ! videoconvert ! - * "video/x-raw,format=I420,width=1920,height=1080,framerate=25/1 ! " "queue ! " + * "video/x-raw,format=I420,width=1280,height=720,framerate=25/1 ! " "queue ! " * "x265enc speed-preset=veryfast bitrate=512 tune=zerolatency ! " * "video/x-h265,stream-format=byte-stream,alignment=au,profile=main ! appsink sync=TRUE " * "emit-signals=TRUE name=appsink-video audiotestsrc wave=triangle is-live=TRUE ! " @@ -198,15 +198,24 @@ PVOID sendGstreamerAudioVideo(PVOID args) case SAMPLE_STREAMING_VIDEO_ONLY: switch (pSampleConfiguration->srcType) { case TEST_SOURCE: { - senderPipeline = gst_parse_launch( - "videotestsrc pattern=ball is-live=TRUE ! " - "queue ! videoconvert ! videoscale ! video/x-raw,width=1280,height=720 ! " - "clockoverlay halignment=right valignment=top time-format=\"%Y-%m-%d %H:%M:%S\" ! " - "videorate ! video/x-raw,framerate=25/1 ! " - "x264enc name=sampleVideoEncoder bframes=0 speed-preset=veryfast bitrate=512 byte-stream=TRUE tune=zerolatency ! " - "video/x-h264,stream-format=byte-stream,alignment=au,profile=baseline ! " - "appsink sync=TRUE emit-signals=TRUE name=appsink-video", - &error); + if (pSampleConfiguration->videoCodec == RTC_CODEC_H265) { + senderPipeline = gst_parse_launch("videotestsrc pattern=ball is-live=TRUE ! timeoverlay ! queue ! videoconvert ! " + "video/x-raw,width=1280,height=720,framerate=25/1 ! queue ! " + "x265enc speed-preset=veryfast bitrate=512 tune=zerolatency ! " + "video/x-h265,stream-format=byte-stream,alignment=au,profile=main ! appsink sync=TRUE " + "emit-signals=TRUE name=appsink-video", + &error); + } else { + senderPipeline = gst_parse_launch( + "videotestsrc pattern=ball is-live=TRUE ! " + "queue ! videoconvert ! videoscale ! video/x-raw,width=1280,height=720 ! " + "clockoverlay halignment=right valignment=top time-format=\"%Y-%m-%d %H:%M:%S\" ! " + "videorate ! video/x-raw,framerate=25/1 ! " + "x264enc name=sampleVideoEncoder bframes=0 speed-preset=veryfast bitrate=512 byte-stream=TRUE tune=zerolatency ! " + "video/x-h264,stream-format=byte-stream,alignment=au,profile=baseline ! " + "appsink sync=TRUE emit-signals=TRUE name=appsink-video", + &error); + } break; } case DEVICE_SOURCE: { @@ -243,16 +252,42 @@ PVOID sendGstreamerAudioVideo(PVOID args) case SAMPLE_STREAMING_AUDIO_VIDEO: switch (pSampleConfiguration->srcType) { case TEST_SOURCE: { - senderPipeline = gst_parse_launch( - "videotestsrc pattern=ball is-live=TRUE ! " - "queue ! videorate ! videoscale ! videoconvert ! video/x-raw,width=1280,height=720,framerate=25/1 ! " - "clockoverlay halignment=right valignment=top time-format=\"%Y-%m-%d %H:%M:%S\" ! " - "x264enc name=sampleVideoEncoder bframes=0 speed-preset=veryfast bitrate=512 byte-stream=TRUE tune=zerolatency ! " - "video/x-h264,stream-format=byte-stream,alignment=au,profile=baseline ! " - "appsink sync=TRUE emit-signals=TRUE name=appsink-video audiotestsrc wave=ticks is-live=TRUE ! " - "queue leaky=2 max-size-buffers=400 ! audioconvert ! audioresample ! opusenc name=sampleAudioEncoder ! " - "audio/x-opus,rate=48000,channels=2 ! appsink sync=TRUE emit-signals=TRUE name=appsink-audio", - &error); + if (pSampleConfiguration->videoCodec == RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE && + pSampleConfiguration->audioCodec == RTC_CODEC_OPUS) { + senderPipeline = gst_parse_launch( + "videotestsrc pattern=ball is-live=TRUE ! " + "queue ! videorate ! videoscale ! videoconvert ! video/x-raw,width=1280,height=720,framerate=25/1 ! " + "clockoverlay halignment=right valignment=top time-format=\"%Y-%m-%d %H:%M:%S\" ! " + "x264enc name=sampleVideoEncoder bframes=0 speed-preset=veryfast bitrate=512 byte-stream=TRUE tune=zerolatency ! " + "video/x-h264,stream-format=byte-stream,alignment=au,profile=baseline ! " + "appsink sync=TRUE emit-signals=TRUE name=appsink-video audiotestsrc wave=ticks is-live=TRUE ! " + "queue leaky=2 max-size-buffers=400 ! audioconvert ! audioresample ! opusenc name=sampleAudioEncoder ! " + "audio/x-opus,rate=48000,channels=2 ! appsink sync=TRUE emit-signals=TRUE name=appsink-audio", + &error); + } else if (pSampleConfiguration->videoCodec == RTC_CODEC_H265 && pSampleConfiguration->audioCodec == RTC_CODEC_OPUS) { + senderPipeline = + gst_parse_launch("videotestsrc pattern=ball is-live=TRUE ! timeoverlay ! queue ! videoconvert ! " + "video/x-raw,width=1280,height=720,framerate=25/1 ! queue ! " + "x265enc speed-preset=veryfast bitrate=512 tune=zerolatency ! " + "video/x-h265,stream-format=byte-stream,alignment=au,profile=main ! appsink sync=TRUE " + "emit-signals=TRUE name=appsink-video audiotestsrc is-live=TRUE ! " + "queue leaky=2 max-size-buffers=400 ! audioconvert ! audioresample ! opusenc ! " + "audio/x-opus,rate=48000,channels=2 ! appsink sync=TRUE emit-signals=TRUE name=appsink-audio", + &error); + + } else if (pSampleConfiguration->videoCodec == RTC_CODEC_H265 && pSampleConfiguration->audioCodec == RTC_CODEC_AAC) { + senderPipeline = + gst_parse_launch("videotestsrc pattern=ball is-live=TRUE ! timeoverlay ! queue ! videoconvert ! " + "video/x-raw,format=I420,width=1280,height=720,framerate=25/1 ! queue ! " + "x265enc speed-preset=veryfast bitrate=512 tune=zerolatency ! " + "video/x-h265,stream-format=byte-stream,alignment=au,profile=main ! appsink sync=TRUE " + "emit-signals=TRUE name=appsink-video audiotestsrc wave=triangle is-live=TRUE ! " + "queue leaky=2 max-size-buffers=400 ! audioconvert ! audioresample ! faac ! " + "capsfilter caps=audio/mpeg,mpegversion=4,stream-format=adts,base-profile=lc,channels=2,rate=48000 ! " + "appsink sync=TRUE emit-signals=TRUE name=appsink-audio", + &error); + } + // TODO: test and add more such combinations break; } case DEVICE_SOURCE: { @@ -345,6 +380,8 @@ INT32 main(INT32 argc, CHAR* argv[]) STATUS retStatus = STATUS_SUCCESS; PSampleConfiguration pSampleConfiguration = NULL; PCHAR pChannelName; + RTC_CODEC audioCodec = RTC_CODEC_OPUS; + RTC_CODEC videoCodec = RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE; SET_INSTRUMENTED_ALLOCATORS(); UINT32 logLevel = setLogLevel(); @@ -360,8 +397,24 @@ INT32 main(INT32 argc, CHAR* argv[]) CHK_STATUS(createSampleConfiguration(pChannelName, SIGNALING_CHANNEL_ROLE_TYPE_MASTER, TRUE, TRUE, logLevel, &pSampleConfiguration)); + if (argc > 3 && STRCMP(argv[3], "testsrc") == 0) { + if (argc > 4) { + if (!STRCMP(argv[4], AUDIO_CODEC_NAME_AAC)) { + audioCodec = RTC_CODEC_AAC; + } + } + + if (argc > 5) { + if (!STRCMP(argv[5], VIDEO_CODEC_NAME_H265)) { + videoCodec = RTC_CODEC_H265; + } + } + } + pSampleConfiguration->videoSource = sendGstreamerAudioVideo; pSampleConfiguration->mediaType = SAMPLE_STREAMING_VIDEO_ONLY; + pSampleConfiguration->audioCodec = audioCodec; + pSampleConfiguration->videoCodec = videoCodec; #ifdef ENABLE_DATA_CHANNEL pSampleConfiguration->onDataChannel = onDataChannel; diff --git a/samples/kvsWebRTCClientViewer.c b/samples/kvsWebRTCClientViewer.c index b627dd4f93..bc60e12cad 100644 --- a/samples/kvsWebRTCClientViewer.c +++ b/samples/kvsWebRTCClientViewer.c @@ -39,6 +39,8 @@ INT32 main(INT32 argc, CHAR* argv[]) SignalingMessage message; PSampleConfiguration pSampleConfiguration = NULL; PSampleStreamingSession pSampleStreamingSession = NULL; + RTC_CODEC audioCodec = RTC_CODEC_OPUS; + RTC_CODEC videoCodec = RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE; BOOL locked = FALSE; PCHAR pChannelName; CHAR clientId[256]; @@ -57,8 +59,32 @@ INT32 main(INT32 argc, CHAR* argv[]) pChannelName = argc > 1 ? argv[1] : SAMPLE_CHANNEL_NAME; #endif + if (argc > 2) { + if (!STRCMP(argv[2], AUDIO_CODEC_NAME_AAC)) { + audioCodec = RTC_CODEC_AAC; + } else if (!STRCMP(argv[2], AUDIO_CODEC_NAME_ALAW)) { + audioCodec = RTC_CODEC_ALAW; + } else if (!STRCMP(argv[2], AUDIO_CODEC_NAME_MULAW)) { + audioCodec = RTC_CODEC_MULAW; + } else { + DLOGI("[KVS Viewer] Defaulting to Opus audio codec"); + } + } + + if (argc > 3) { + if (!STRCMP(argv[3], VIDEO_CODEC_NAME_H265)) { + videoCodec = RTC_CODEC_H265; + } else if (!STRCMP(argv[3], VIDEO_CODEC_NAME_VP8)) { + videoCodec = RTC_CODEC_VP8; + } else { + DLOGI("[KVS Viewer] Defaulting to H264 video codec"); + } + } + CHK_STATUS(createSampleConfiguration(pChannelName, SIGNALING_CHANNEL_ROLE_TYPE_VIEWER, TRUE, TRUE, logLevel, &pSampleConfiguration)); pSampleConfiguration->mediaType = SAMPLE_STREAMING_AUDIO_VIDEO; + pSampleConfiguration->audioCodec = audioCodec; + pSampleConfiguration->videoCodec = videoCodec; // Initialize KVS WebRTC. This must be done before anything else, and must only be done once. CHK_STATUS(initKvsWebRtc()); diff --git a/samples/kvsWebRTCClientViewerGstSample.c b/samples/kvsWebRTCClientViewerGstSample.c index 5fa167bbcc..18ac6625a4 100644 --- a/samples/kvsWebRTCClientViewerGstSample.c +++ b/samples/kvsWebRTCClientViewerGstSample.c @@ -26,7 +26,7 @@ VOID dataChannelOnOpenCallback(UINT64 customData, PRtcDataChannel pDataChannel) // Sending first message to the master over the data channel retStatus = dataChannelSend(pDataChannel, FALSE, (PBYTE) VIEWER_DATA_CHANNEL_MESSAGE, STRLEN(VIEWER_DATA_CHANNEL_MESSAGE)); if (retStatus != STATUS_SUCCESS) { - DLOGI("[KVS Viewer] dataChannelSend(): operation returned status code: 0x%08x ", retStatus); + DLOGI("[KVS Gstreamer Viewer] dataChannelSend(): operation returned status code: 0x%08x ", retStatus); } } #endif @@ -39,6 +39,8 @@ INT32 main(INT32 argc, CHAR* argv[]) SignalingMessage message; PSampleConfiguration pSampleConfiguration = NULL; PSampleStreamingSession pSampleStreamingSession = NULL; + RTC_CODEC audioCodec = RTC_CODEC_OPUS; + RTC_CODEC videoCodec = RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE; BOOL locked = FALSE; PCHAR pChannelName; CHAR clientId[256]; @@ -57,27 +59,45 @@ INT32 main(INT32 argc, CHAR* argv[]) pChannelName = argc > 1 ? argv[1] : SAMPLE_CHANNEL_NAME; #endif + if (argc > 3) { + if (!STRCMP(argv[3], AUDIO_CODEC_NAME_AAC)) { + audioCodec = RTC_CODEC_AAC; + } else { + DLOGI("[KVS Gstreamer Viewer] Defaulting to Opus audio codec"); + } + } + + if (argc > 4) { + if (!STRCMP(argv[4], VIDEO_CODEC_NAME_H265)) { + videoCodec = RTC_CODEC_H265; + } else { + DLOGI("[KVS Gstreamer Viewer] Defaulting to H264 video codec"); + } + } + CHK_STATUS(createSampleConfiguration(pChannelName, SIGNALING_CHANNEL_ROLE_TYPE_VIEWER, TRUE, TRUE, logLevel, &pSampleConfiguration)); pSampleConfiguration->mediaType = SAMPLE_STREAMING_AUDIO_VIDEO; pSampleConfiguration->receiveAudioVideoSource = receiveGstreamerAudioVideo; + pSampleConfiguration->audioCodec = audioCodec; + pSampleConfiguration->videoCodec = videoCodec; if (argc > 2) { if (STRCMP(argv[2], "video-only") == 0) { pSampleConfiguration->mediaType = SAMPLE_STREAMING_VIDEO_ONLY; - DLOGI("[KVS Gstreamer Master] Streaming video only"); + DLOGI("[KVS Gstreamer Viewer] Streaming video only"); } else if (STRCMP(argv[2], "audio-video") == 0) { pSampleConfiguration->mediaType = SAMPLE_STREAMING_AUDIO_VIDEO; - DLOGI("[KVS Gstreamer Master] Streaming audio and video"); + DLOGI("[KVS Gstreamer Viewer] Streaming audio and video"); } else { - DLOGI("[KVS Gstreamer Master] Unrecognized streaming type. Default to video-only"); + DLOGI("[KVS Gstreamer Viewer] Unrecognized streaming type. Default to video-only"); } } else { - DLOGI("[KVS Gstreamer Master] Streaming video only"); + DLOGI("[KVS Gstreamer Viewer] Streaming video only"); } // Initialize KVS WebRTC. This must be done before anything else, and must only be done once. CHK_STATUS(initKvsWebRtc()); - DLOGI("[KVS Viewer] KVS WebRTC initialization completed successfully"); + DLOGI("[KVS Gstreamer Viewer] KVS WebRTC initialization completed successfully"); #ifdef ENABLE_DATA_CHANNEL pSampleConfiguration->onDataChannel = onDataChannel; @@ -85,13 +105,13 @@ INT32 main(INT32 argc, CHAR* argv[]) SPRINTF(clientId, "%s_%u", SAMPLE_VIEWER_CLIENT_ID, RAND() % MAX_UINT32); CHK_STATUS(initSignaling(pSampleConfiguration, clientId)); - DLOGI("[KVS Viewer] Signaling client connection established"); + DLOGI("[KVS Gstreamer Viewer] Signaling client connection established"); // Initialize streaming session MUTEX_LOCK(pSampleConfiguration->sampleConfigurationObjLock); locked = TRUE; CHK_STATUS(createSampleStreamingSession(pSampleConfiguration, NULL, FALSE, &pSampleStreamingSession)); - DLOGI("[KVS Viewer] Creating streaming session...completed"); + DLOGI("[KVS Gstreamer Viewer] Creating streaming session...completed"); pSampleConfiguration->sampleStreamingSessionList[pSampleConfiguration->streamingSessionCount++] = pSampleStreamingSession; MUTEX_UNLOCK(pSampleConfiguration->sampleConfigurationObjLock); @@ -101,10 +121,10 @@ INT32 main(INT32 argc, CHAR* argv[]) offerSessionDescriptionInit.useTrickleIce = pSampleStreamingSession->remoteCanTrickleIce; CHK_STATUS(setLocalDescription(pSampleStreamingSession->pPeerConnection, &offerSessionDescriptionInit)); - DLOGI("[KVS Viewer] Completed setting local description"); + DLOGI("[KVS Gstreamer Viewer] Completed setting local description"); if (!pSampleConfiguration->trickleIce) { - DLOGI("[KVS Viewer] Non trickle ice. Wait for Candidate collection to complete"); + DLOGI("[KVS Gstreamer Viewer] Non trickle ice. Wait for Candidate collection to complete"); MUTEX_LOCK(pSampleConfiguration->sampleConfigurationObjLock); locked = TRUE; @@ -117,17 +137,17 @@ INT32 main(INT32 argc, CHAR* argv[]) MUTEX_UNLOCK(pSampleConfiguration->sampleConfigurationObjLock); locked = FALSE; - DLOGI("[KVS Viewer] Candidate collection completed"); + DLOGI("[KVS Gstreamer Viewer] Candidate collection completed"); } CHK_STATUS(createOffer(pSampleStreamingSession->pPeerConnection, &offerSessionDescriptionInit)); - DLOGI("[KVS Viewer] Offer creation successful"); + DLOGI("[KVS Gstreamer Viewer] Offer creation successful"); - DLOGI("[KVS Viewer] Generating JSON of session description...."); + DLOGI("[KVS Gstreamer Viewer] Generating JSON of session description...."); CHK_STATUS(serializeSessionDescriptionInit(&offerSessionDescriptionInit, NULL, &buffLen)); if (buffLen >= SIZEOF(message.payload)) { - DLOGE("[KVS Viewer] serializeSessionDescriptionInit(): operation returned status code: 0x%08x ", STATUS_INVALID_OPERATION); + DLOGE("[KVS Gstreamer Viewer] serializeSessionDescriptionInit(): operation returned status code: 0x%08x ", STATUS_INVALID_OPERATION); retStatus = STATUS_INVALID_OPERATION; goto CleanUp; } @@ -148,11 +168,11 @@ INT32 main(INT32 argc, CHAR* argv[]) // Creating a new datachannel on the peer connection of the existing sample streaming session CHK_STATUS(createDataChannel(pPeerConnection, pChannelName, NULL, &pDataChannel)); - DLOGI("[KVS Viewer] Creating data channel...completed"); + DLOGI("[KVS Gstreamer Viewer] Creating data channel...completed"); // Setting a callback for when the data channel is open CHK_STATUS(dataChannelOnOpen(pDataChannel, (UINT64) &datachannelLocalOpenCount, dataChannelOnOpenCallback)); - DLOGI("[KVS Viewer] Data Channel open now..."); + DLOGI("[KVS Gstreamer Viewer] Data Channel open now..."); #endif // Block until interrupted @@ -163,10 +183,10 @@ INT32 main(INT32 argc, CHAR* argv[]) CleanUp: if (retStatus != STATUS_SUCCESS) { - DLOGE("[KVS Viewer] Terminated with status code 0x%08x", retStatus); + DLOGE("[KVS Gstreamer Viewer] Terminated with status code 0x%08x", retStatus); } - DLOGI("[KVS Viewer] Cleaning up...."); + DLOGI("[KVS Gstreamer Viewer] Cleaning up...."); if (locked) { MUTEX_UNLOCK(pSampleConfiguration->sampleConfigurationObjLock); @@ -178,15 +198,15 @@ INT32 main(INT32 argc, CHAR* argv[]) if (pSampleConfiguration != NULL) { retStatus = freeSignalingClient(&pSampleConfiguration->signalingClientHandle); if (retStatus != STATUS_SUCCESS) { - DLOGE("[KVS Viewer] freeSignalingClient(): operation returned status code: 0x%08x ", retStatus); + DLOGE("[KVS Gstreamer Viewer] freeSignalingClient(): operation returned status code: 0x%08x ", retStatus); } retStatus = freeSampleConfiguration(&pSampleConfiguration); if (retStatus != STATUS_SUCCESS) { - DLOGE("[KVS Viewer] freeSampleConfiguration(): operation returned status code: 0x%08x ", retStatus); + DLOGE("[KVS Gstreamer Viewer] freeSampleConfiguration(): operation returned status code: 0x%08x ", retStatus); } } - DLOGI("[KVS Viewer] Cleanup done"); + DLOGI("[KVS Gstreamer Viewer] Cleanup done"); RESET_INSTRUMENTED_ALLOCATORS(); diff --git a/src/source/Include_i.h b/src/source/Include_i.h index 1c9ba3510c..2b9e2bfc28 100644 --- a/src/source/Include_i.h +++ b/src/source/Include_i.h @@ -159,6 +159,7 @@ STATUS generateJSONSafeString(PCHAR, UINT32); #include "PeerConnection/DataChannel.h" #include "Rtp/Codecs/RtpVP8Payloader.h" #include "Rtp/Codecs/RtpH264Payloader.h" +#include "Rtp/Codecs/RtpH265Payloader.h" #include "Rtp/Codecs/RtpOpusPayloader.h" #include "Rtp/Codecs/RtpG711Payloader.h" #include "Metrics/Metrics.h" diff --git a/src/source/PeerConnection/PeerConnection.c b/src/source/PeerConnection/PeerConnection.c index 2501f422ca..965eaac757 100644 --- a/src/source/PeerConnection/PeerConnection.c +++ b/src/source/PeerConnection/PeerConnection.c @@ -1563,6 +1563,10 @@ STATUS addTransceiver(PRtcPeerConnection pPeerConnection, PRtcMediaStreamTrack p depayFunc = depayVP8FromRtpPayload; clockRate = VIDEO_CLOCKRATE; break; + case RTC_CODEC_H265: + depayFunc = depayH265FromRtpPayload; + clockRate = VIDEO_CLOCKRATE; + break; default: CHK(FALSE, STATUS_NOT_IMPLEMENTED); @@ -1608,8 +1612,21 @@ STATUS addSupportedCodec(PRtcPeerConnection pPeerConnection, RTC_CODEC rtcCodec) CHK(pKvsPeerConnection != NULL, STATUS_NULL_ARG); - CHK_STATUS(hashTablePut(pKvsPeerConnection->pCodecTable, rtcCodec, 0)); - + if (rtcCodec == RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE) { + CHK_STATUS(hashTablePut(pKvsPeerConnection->pCodecTable, rtcCodec, DEFAULT_PAYLOAD_H264)); + } else if (rtcCodec == RTC_CODEC_VP8) { + CHK_STATUS(hashTablePut(pKvsPeerConnection->pCodecTable, rtcCodec, DEFAULT_PAYLOAD_VP8)); + } else if (rtcCodec == RTC_CODEC_H265) { + CHK_STATUS(hashTablePut(pKvsPeerConnection->pCodecTable, rtcCodec, DEFAULT_PAYLOAD_H265)); + } else if (rtcCodec == RTC_CODEC_OPUS) { + CHK_STATUS(hashTablePut(pKvsPeerConnection->pCodecTable, rtcCodec, DEFAULT_PAYLOAD_OPUS)); + } else if (rtcCodec == RTC_CODEC_MULAW) { + CHK_STATUS(hashTablePut(pKvsPeerConnection->pCodecTable, rtcCodec, DEFAULT_PAYLOAD_MULAW)); + } else if (rtcCodec == RTC_CODEC_ALAW) { + CHK_STATUS(hashTablePut(pKvsPeerConnection->pCodecTable, rtcCodec, DEFAULT_PAYLOAD_ALAW)); + } else { + CHK_STATUS(hashTablePut(pKvsPeerConnection->pCodecTable, rtcCodec, 0)); + } CleanUp: LEAVES(); diff --git a/src/source/PeerConnection/PeerConnection.h b/src/source/PeerConnection/PeerConnection.h index 21bcae88a9..ee12dabc1a 100644 --- a/src/source/PeerConnection/PeerConnection.h +++ b/src/source/PeerConnection/PeerConnection.h @@ -45,6 +45,7 @@ extern "C" { typedef enum { RTC_RTX_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE = 1, RTC_RTX_CODEC_VP8 = 2, + RTC_RTX_CODEC_H265 = 3, } RTX_CODEC; typedef struct { diff --git a/src/source/PeerConnection/Rtp.c b/src/source/PeerConnection/Rtp.c index a1757cac6a..a78e116688 100644 --- a/src/source/PeerConnection/Rtp.c +++ b/src/source/PeerConnection/Rtp.c @@ -257,6 +257,11 @@ STATUS writeFrame(PRtcRtpTransceiver pRtcRtpTransceiver, PFrame pFrame) rtpTimestamp = CONVERT_TIMESTAMP_TO_RTP(VIDEO_CLOCKRATE, pFrame->presentationTs); break; + case RTC_CODEC_H265: + rtpPayloadFunc = createPayloadForH265; + rtpTimestamp = CONVERT_TIMESTAMP_TO_RTP(VIDEO_CLOCKRATE, pFrame->presentationTs); + break; + case RTC_CODEC_OPUS: rtpPayloadFunc = createPayloadForOpus; rtpTimestamp = CONVERT_TIMESTAMP_TO_RTP(OPUS_CLOCKRATE, pFrame->presentationTs); diff --git a/src/source/PeerConnection/SessionDescription.c b/src/source/PeerConnection/SessionDescription.c index b07b4cd501..215bdb6a96 100644 --- a/src/source/PeerConnection/SessionDescription.c +++ b/src/source/PeerConnection/SessionDescription.c @@ -138,6 +138,7 @@ STATUS setPayloadTypesForOffer(PHashTable codecTable) CHK_STATUS(hashTableUpsert(codecTable, RTC_CODEC_VP8, DEFAULT_PAYLOAD_VP8)); CHK_STATUS(hashTableUpsert(codecTable, RTC_CODEC_OPUS, DEFAULT_PAYLOAD_OPUS)); CHK_STATUS(hashTableUpsert(codecTable, RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE, DEFAULT_PAYLOAD_H264)); + CHK_STATUS(hashTableUpsert(codecTable, RTC_CODEC_H265, DEFAULT_PAYLOAD_H265)); CleanUp: return retStatus; @@ -187,7 +188,12 @@ STATUS setPayloadTypesFromOffer(PHashTable codecTable, PHashTable rtxTable, PSes for (currentAttribute = 0; currentAttribute < pMediaDescription->mediaAttributesCount; currentAttribute++) { attributeValue = pMediaDescription->sdpAttributes[currentAttribute].attributeValue; - + CHK_STATUS(hashTableContains(codecTable, RTC_CODEC_H265, &supportCodec)); + if (supportCodec && (end = STRSTR(attributeValue, H265_VALUE)) != NULL) { + CHK_STATUS(STRTOUI64(attributeValue, end - 1, 10, &parsedPayloadType)); + DLOGV("Found H265 payload type %" PRId64 ".", parsedPayloadType); + CHK_STATUS(hashTableUpsert(codecTable, RTC_CODEC_H265, parsedPayloadType)); + } CHK_STATUS(hashTableContains(codecTable, RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE, &supportCodec)); if (supportCodec && (end = STRSTR(attributeValue, H264_VALUE)) != NULL) { CHK_STATUS(STRTOUI64(attributeValue, end - 1, 10, &parsedPayloadType)); @@ -241,6 +247,15 @@ STATUS setPayloadTypesFromOffer(PHashTable codecTable, PHashTable rtxTable, PSes fmtpVal = aptFmtVal >> 8u; aptVal = aptFmtVal & 0xFFu; + CHK_STATUS(hashTableContains(codecTable, RTC_CODEC_H265, &supportCodec)); + if (supportCodec) { + CHK_STATUS(hashTableGet(codecTable, RTC_CODEC_H265, &hashmapPayloadType)); + if (aptVal == hashmapPayloadType) { + CHK_STATUS(hashTableUpsert(rtxTable, RTC_RTX_CODEC_H265, fmtpVal)); + DLOGV("h265 found apt type %" PRId64 " for fmtp %" PRId64, aptVal, fmtpVal); + } + } + CHK_STATUS(hashTableContains(codecTable, RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE, &supportCodec)); if (supportCodec) { CHK_STATUS(hashTableGet(codecTable, RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE, &hashmapPayloadType)); @@ -420,12 +435,16 @@ STATUS populateSingleMediaSection(PKvsPeerConnection pKvsPeerConnection, PKvsRtp currentFmtp = fmtpForPayloadType(payloadType, pRemoteSessionDescription); } } + if (pRtcMediaStreamTrack->kind == MEDIA_STREAM_TRACK_KIND_VIDEO) { if (pRtcMediaStreamTrack->codec == RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE) { retStatus = hashTableGet(pKvsPeerConnection->pRtxTable, RTC_RTX_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE, &rtxPayloadType); } else if (pRtcMediaStreamTrack->codec == RTC_CODEC_VP8) { retStatus = hashTableGet(pKvsPeerConnection->pRtxTable, RTC_RTX_CODEC_VP8, &rtxPayloadType); + } else if (pRtcMediaStreamTrack->codec == RTC_CODEC_H265) { + retStatus = hashTableGet(pKvsPeerConnection->pRtxTable, RTC_RTX_CODEC_H265, &rtxPayloadType); + payloadType = DEFAULT_PAYLOAD_H265; } else { retStatus = STATUS_HASH_KEY_NOT_PRESENT; } @@ -731,6 +750,50 @@ STATUS populateSingleMediaSection(PKvsPeerConnection pKvsPeerConnection, PKvsRtp SIZEOF(pSdpMediaDescription->sdpAttributes[attributeCount].attributeValue), "%" PRId64 " " ALAW_VALUE, payloadType); CHK_ERR(amountWritten > 0, STATUS_INTERNAL_ERROR, "Full ALAW rtpmap could not be written"); attributeCount++; + } else if (pRtcMediaStreamTrack->codec == RTC_CODEC_H265) { + if (pKvsPeerConnection->isOffer) { + currentFmtp = DEFAULT_H265_FMTP; + } + STRCPY(pSdpMediaDescription->sdpAttributes[attributeCount].attributeName, "rtpmap"); + amountWritten = SNPRINTF(pSdpMediaDescription->sdpAttributes[attributeCount].attributeValue, + SIZEOF(pSdpMediaDescription->sdpAttributes[attributeCount].attributeValue), "%" PRId64 " H265/90000", payloadType); + CHK_ERR(amountWritten > 0, STATUS_INTERNAL_ERROR, "Full H265 rtpmap could not be written"); + attributeCount++; + + STRCPY(pSdpMediaDescription->sdpAttributes[attributeCount].attributeName, "rtcp-fb"); + amountWritten = SNPRINTF(pSdpMediaDescription->sdpAttributes[attributeCount].attributeValue, + SIZEOF(pSdpMediaDescription->sdpAttributes[attributeCount].attributeValue), "%" PRId64 " nack", payloadType); + CHK_ERR(amountWritten > 0, STATUS_INTERNAL_ERROR, "Full H265 rtcp-fb nack value could not be written"); + amountWritten = SNPRINTF(pSdpMediaDescription->sdpAttributes[attributeCount].attributeValue, + SIZEOF(pSdpMediaDescription->sdpAttributes[attributeCount].attributeValue), "%" PRId64 " nack pli", payloadType); + CHK_ERR(amountWritten > 0, STATUS_INTERNAL_ERROR, "Full H265 rtcp-fb nack-pli value could not be written"); + attributeCount++; + + // TODO: If level asymmetry is allowed, consider sending back DEFAULT_H265_FMTP instead of the received fmtp value. + if (currentFmtp != NULL) { + STRCPY(pSdpMediaDescription->sdpAttributes[attributeCount].attributeName, "fmtp"); + amountWritten = + SNPRINTF(pSdpMediaDescription->sdpAttributes[attributeCount].attributeValue, + SIZEOF(pSdpMediaDescription->sdpAttributes[attributeCount].attributeValue), "%" PRId64 " %s", payloadType, currentFmtp); + CHK_ERR(amountWritten > 0, STATUS_INTERNAL_ERROR, "Full H265 fmtp value could not be written"); + attributeCount++; + } + + if (containRtx) { + STRCPY(pSdpMediaDescription->sdpAttributes[attributeCount].attributeName, "rtpmap"); + amountWritten = + SNPRINTF(pSdpMediaDescription->sdpAttributes[attributeCount].attributeValue, + SIZEOF(pSdpMediaDescription->sdpAttributes[attributeCount].attributeValue), "%" PRId64 " " RTX_VALUE, rtxPayloadType); + CHK_ERR(amountWritten > 0, STATUS_INTERNAL_ERROR, "Full H265 rtpmap (with rtx) could not be written"); + attributeCount++; + + STRCPY(pSdpMediaDescription->sdpAttributes[attributeCount].attributeName, "fmtp"); + amountWritten = SNPRINTF(pSdpMediaDescription->sdpAttributes[attributeCount].attributeValue, + SIZEOF(pSdpMediaDescription->sdpAttributes[attributeCount].attributeValue), "%" PRId64 " apt=%" PRId64 "", + rtxPayloadType, payloadType); + CHK_ERR(amountWritten > 0, STATUS_INTERNAL_ERROR, "Full H265 fmtp apt value (with rtx) could not be written"); + attributeCount++; + } } else if (pRtcMediaStreamTrack->codec == RTC_CODEC_UNKNOWN) { CHK_STATUS(hashTableGet(pUnknownCodecRtpmapTable, unknownCodecHashTableKey, (PUINT64) &rtpMapValue)); STRCPY(pSdpMediaDescription->sdpAttributes[attributeCount].attributeName, "rtpmap"); @@ -1218,6 +1281,9 @@ STATUS findTransceiversByRemoteDescription(PKvsPeerConnection pKvsPeerConnection } else if (STRSTR(attributeValue, VP8_VALUE) != NULL) { supportCodec = TRUE; rtcCodec = RTC_CODEC_VP8; + } else if (STRSTR(attributeValue, H265_VALUE) != NULL) { + supportCodec = TRUE; + rtcCodec = RTC_CODEC_H265; } else { supportCodec = FALSE; } @@ -1348,7 +1414,8 @@ STATUS setReceiversSsrc(PSessionDescription pRemoteSessionDescription, PDoubleLi CHK_STATUS(doubleListGetNodeData(pCurNode, &data)); pKvsRtpTransceiver = (PKvsRtpTransceiver) data; codec = pKvsRtpTransceiver->sender.track.codec; - isVideoCodec = (codec == RTC_CODEC_VP8 || codec == RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE); + isVideoCodec = (codec == RTC_CODEC_VP8 || codec == RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE || + codec == RTC_CODEC_H265); isAudioCodec = (codec == RTC_CODEC_MULAW || codec == RTC_CODEC_ALAW || codec == RTC_CODEC_OPUS); if (pKvsRtpTransceiver->jitterBufferSsrc == 0 && diff --git a/src/source/PeerConnection/SessionDescription.h b/src/source/PeerConnection/SessionDescription.h index 74f1a1dda0..a7e5ac4903 100644 --- a/src/source/PeerConnection/SessionDescription.h +++ b/src/source/PeerConnection/SessionDescription.h @@ -30,6 +30,7 @@ extern "C" { #define MID_KEY "mid" #define H264_VALUE "H264/90000" +#define H265_VALUE "H265/90000" #define OPUS_VALUE "opus/48000" #define VP8_VALUE "VP8/90000" #define MULAW_VALUE "PCMU/8000" @@ -45,11 +46,15 @@ extern "C" { #define DEFAULT_PAYLOAD_OPUS (UINT64) 111 #define DEFAULT_PAYLOAD_VP8 (UINT64) 96 #define DEFAULT_PAYLOAD_H264 (UINT64) 125 +#define DEFAULT_PAYLOAD_H265 (UINT64) 127 #define DEFAULT_PAYLOAD_MULAW_STR (PCHAR) "0" #define DEFAULT_PAYLOAD_ALAW_STR (PCHAR) "8" -#define DEFAULT_H264_FMTP (PCHAR) "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f" +#define DEFAULT_H264_FMTP (PCHAR) "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f" +#define DEFAULT_H265_FMTP \ + (PCHAR) "profile-space=0;profile-id=0;tier-flag=0;level-id=0;interop-constraints=000000000000;sprop-vps=QAEMAf//" \ + "AIAAAAMAAAMAAAMAAAMAALUCQA==;sprop-sps=QgEBAIAAAAMAAAMAAAMAAAMAAKACgIAtH+W1kkbQzkkktySqSfKSyA==;sprop-pps=RAHBpVgeSA==" #define DEFAULT_OPUS_FMTP (PCHAR) "minptime=10;useinbandfec=1" #define H264_PROFILE_42E01F 0x42e01f // profile-level-id: diff --git a/src/source/Rtp/Codecs/RtpH264Payloader.c b/src/source/Rtp/Codecs/RtpH264Payloader.c index adc7ec3776..1b2ea942d2 100644 --- a/src/source/Rtp/Codecs/RtpH264Payloader.c +++ b/src/source/Rtp/Codecs/RtpH264Payloader.c @@ -78,15 +78,16 @@ STATUS getNextNaluLength(PBYTE nalus, UINT32 nalusLength, PUINT32 pStart, PUINT3 ENTERS(); STATUS retStatus = STATUS_SUCCESS; - UINT32 zeroCount = 0, offset; + UINT32 zeroCount = 0, offset = 0; BOOL naluFound = FALSE; PBYTE pCurrent = NULL; CHK(nalus != NULL && pStart != NULL && pNaluLength != NULL, STATUS_NULL_ARG); // Annex-B Nalu will have 0x000000001 or 0x000001 start code, at most 4 bytes - for (offset = 0; offset < 4 && offset < nalusLength && nalus[offset] == 0; offset++) - ; + while (offset < 4 && offset < nalusLength && nalus[offset] == 0) { + offset++; + } CHK(offset < nalusLength && offset < 4 && offset >= 2 && nalus[offset] == 1, STATUS_RTP_INVALID_NALU); *pStart = ++offset; @@ -146,7 +147,6 @@ STATUS createPayloadFromNalu(UINT32 mtu, PBYTE nalu, UINT32 naluLength, PPayload BOOL sizeCalculationOnly = (pPayloadArray == NULL); CHK(nalu != NULL && filledLength != NULL && filledSubLenSize != NULL, STATUS_NULL_ARG); - sizeCalculationOnly = (pPayloadArray == NULL); CHK(sizeCalculationOnly || (pPayloadArray->payloadSubLength != NULL && pPayloadArray->payloadBuffer != NULL), STATUS_NULL_ARG); CHK(mtu > FU_A_HEADER_SIZE, STATUS_RTP_INPUT_MTU_TOO_SMALL); diff --git a/src/source/Rtp/Codecs/RtpH265Payloader.c b/src/source/Rtp/Codecs/RtpH265Payloader.c new file mode 100644 index 0000000000..b6bb2a87f4 --- /dev/null +++ b/src/source/Rtp/Codecs/RtpH265Payloader.c @@ -0,0 +1,283 @@ +#define LOG_CLASS "RtpH265Payloader" + +#include "../../Include_i.h" + +STATUS createPayloadForH265(UINT32 mtu, PBYTE nalus, UINT32 nalusLength, PBYTE payloadBuffer, PUINT32 pPayloadLength, PUINT32 pPayloadSubLength, + PUINT32 pPayloadSubLenSize) +{ + ENTERS(); + STATUS retStatus = STATUS_SUCCESS; + PBYTE curPtrInNalus = nalus; + UINT32 remainNalusLength = nalusLength; + UINT32 nextNaluLength = 0; + UINT32 startIndex = 0; + UINT32 singlePayloadLength = 0; + UINT32 singlePayloadSubLenSize = 0; + BOOL sizeCalculationOnly = (payloadBuffer == NULL); + PayloadArray payloadArray; + + CHK(nalus != NULL && pPayloadSubLenSize != NULL && pPayloadLength != NULL && (sizeCalculationOnly || pPayloadSubLength != NULL), STATUS_NULL_ARG); + CHK(mtu > H265_FU_HEADER_SIZE, STATUS_RTP_INPUT_MTU_TOO_SMALL); + + if (sizeCalculationOnly) { + payloadArray.payloadLength = 0; + payloadArray.payloadSubLenSize = 0; + payloadArray.maxPayloadLength = 0; + payloadArray.maxPayloadSubLenSize = 0; + } else { + payloadArray.payloadLength = *pPayloadLength; + payloadArray.payloadSubLenSize = *pPayloadSubLenSize; + payloadArray.maxPayloadLength = *pPayloadLength; + payloadArray.maxPayloadSubLenSize = *pPayloadSubLenSize; + } + payloadArray.payloadBuffer = payloadBuffer; + payloadArray.payloadSubLength = pPayloadSubLength; + + do { + CHK_STATUS(getNextNaluLengthH265(curPtrInNalus, remainNalusLength, &startIndex, &nextNaluLength)); + + curPtrInNalus += startIndex; + + remainNalusLength -= startIndex; + + CHK(remainNalusLength != 0, retStatus); + + if (sizeCalculationOnly) { + CHK_STATUS(createPayloadFromNaluH265(mtu, curPtrInNalus, nextNaluLength, NULL, &singlePayloadLength, &singlePayloadSubLenSize)); + payloadArray.payloadLength += singlePayloadLength; + payloadArray.payloadSubLenSize += singlePayloadSubLenSize; + } else { + CHK_STATUS(createPayloadFromNaluH265(mtu, curPtrInNalus, nextNaluLength, &payloadArray, &singlePayloadLength, &singlePayloadSubLenSize)); + payloadArray.payloadBuffer += singlePayloadLength; + payloadArray.payloadSubLength += singlePayloadSubLenSize; + payloadArray.maxPayloadLength -= singlePayloadLength; + payloadArray.maxPayloadSubLenSize -= singlePayloadSubLenSize; + } + + remainNalusLength -= nextNaluLength; + curPtrInNalus += nextNaluLength; + } while (remainNalusLength != 0); + +CleanUp: + if (STATUS_FAILED(retStatus) && sizeCalculationOnly) { + payloadArray.payloadLength = 0; + payloadArray.payloadSubLenSize = 0; + } + + if (pPayloadSubLenSize != NULL && pPayloadLength != NULL) { + *pPayloadLength = payloadArray.payloadLength; + *pPayloadSubLenSize = payloadArray.payloadSubLenSize; + } + + LEAVES(); + return retStatus; +} + +STATUS getNextNaluLengthH265(PBYTE nalus, UINT32 nalusLength, PUINT32 pStart, PUINT32 pNaluLength) +{ + ENTERS(); + + STATUS retStatus = STATUS_SUCCESS; + UINT32 zeroCount = 0, offset = 0; + BOOL naluFound = FALSE; + PBYTE pCurrent = NULL; + + CHK(nalus != NULL && pStart != NULL && pNaluLength != NULL, STATUS_NULL_ARG); + + // Annex-B Nalu will have 0x000000001 or 0x000001 start code, at most 4 bytes + while (offset < 4 && offset < nalusLength && nalus[offset] == 0) { + offset++; + } + + CHK(offset < nalusLength && offset < 4 && offset >= 2 && nalus[offset] == 1, STATUS_RTP_INVALID_NALU); + *pStart = ++offset; + pCurrent = nalus + offset; + + /* Not doing validation on number of consecutive zeros being less than 4 because some device can produce + * data with trailing zeros. */ + while (offset < nalusLength) { + if (*pCurrent == 0) { + /* Maybe next byte is 1 */ + offset++; + pCurrent++; + + } else if (*pCurrent == 1) { + if (*(pCurrent - 1) == 0 && *(pCurrent - 2) == 0) { + zeroCount = *(pCurrent - 3) == 0 ? 3 : 2; + naluFound = TRUE; + break; + } + + /* The jump is always 3 because of the 1 previously matched. + * All the 0's must be after this '1' matched at offset */ + offset += 3; + pCurrent += 3; + } else { + /* Can jump 3 bytes forward */ + offset += 3; + pCurrent += 3; + } + } + *pNaluLength = MIN(offset, nalusLength) - *pStart - (naluFound ? zeroCount : 0); + +CleanUp: + + // As we might hit error often in a "bad" frame scenario, we can't use CHK_LOG_ERR as it will be too frequent + if (STATUS_FAILED(retStatus)) { + DLOGD("Warning: Failed to get the next NALu in H265 payload with 0x%08x", retStatus); + } + + LEAVES(); + return retStatus; +} + +STATUS createPayloadFromNaluH265(UINT32 mtu, PBYTE nalu, UINT32 naluLength, PPayloadArray pPayloadArray, PUINT32 filledLength, + PUINT32 filledSubLenSize) +{ + ENTERS(); + STATUS retStatus = STATUS_SUCCESS; + PBYTE pPayload = NULL; + UINT8 naluType = 0; + UINT32 maxPayloadSize = 0; + UINT32 curPayloadSize = 0; + UINT32 remainingNaluLength = naluLength; + UINT32 payloadLength = 0; + UINT32 payloadSubLenSize = 0; + PBYTE pCurPtrInNalu = NULL; + BOOL sizeCalculationOnly = (pPayloadArray == NULL); + + CHK(nalu != NULL && filledLength != NULL && filledSubLenSize != NULL, STATUS_NULL_ARG); + CHK(sizeCalculationOnly || (pPayloadArray->payloadSubLength != NULL && pPayloadArray->payloadBuffer != NULL), STATUS_NULL_ARG); + CHK(mtu > H265_FU_HEADER_SIZE, STATUS_RTP_INPUT_MTU_TOO_SMALL); + + naluType = (nalu[0] & 0x7E) >> 1; // 6 bits after forbidden zero bit 0x7E(0111 1110) + + if (!sizeCalculationOnly) { + pPayload = pPayloadArray->payloadBuffer; + } + + if (naluLength <= mtu) { + // Single NALU https://www.rfc-editor.org/rfc/rfc7798.html#section-4.4.1 + payloadLength += naluLength; + payloadSubLenSize++; + + if (!sizeCalculationOnly) { + CHK(payloadSubLenSize <= pPayloadArray->maxPayloadSubLenSize && payloadLength <= pPayloadArray->maxPayloadLength, + STATUS_BUFFER_TOO_SMALL); + + MEMCPY(pPayload, nalu, naluLength); + pPayloadArray->payloadSubLength[payloadSubLenSize - 1] = naluLength; + pPayload += pPayloadArray->payloadSubLength[payloadSubLenSize - 1]; + } + } else { + // Fragmentation units: https://www.rfc-editor.org/rfc/rfc7798.html#section-4.4.3 + maxPayloadSize = mtu - H265_FU_HEADER_SIZE; + + // According to the RFC, the first octet is skipped due to redundant information + remainingNaluLength -= 2; + pCurPtrInNalu = nalu + 2; + + while (remainingNaluLength != 0) { + curPayloadSize = MIN(maxPayloadSize, remainingNaluLength); + payloadSubLenSize++; + payloadLength += H265_FU_HEADER_SIZE + curPayloadSize; + + if (!sizeCalculationOnly) { + CHK(payloadSubLenSize <= pPayloadArray->maxPayloadSubLenSize && payloadLength <= pPayloadArray->maxPayloadLength, + STATUS_BUFFER_TOO_SMALL); + + pPayload[0] = (H265_FU_TYPE_ID << 1) | (nalu[0] & 0x81) | (nalu[0] & 0x1); // H265_FU_TYPE_ID indicator is 49 + pPayload[1] = nalu[1] & 0xff; + pPayload[2] = naluType & 0x3f; + if (remainingNaluLength == naluLength - 2) { + pPayload[2] |= (1 << 7); // Set for starting bit + } else if (remainingNaluLength == curPayloadSize) { + pPayload[2] |= (1 << 6); // Set for ending bit + } + + MEMCPY(pPayload + H265_FU_HEADER_SIZE, pCurPtrInNalu, curPayloadSize); + + pPayloadArray->payloadSubLength[payloadSubLenSize - 1] = H265_FU_HEADER_SIZE + curPayloadSize; + pPayload += pPayloadArray->payloadSubLength[payloadSubLenSize - 1]; + } + + pCurPtrInNalu += curPayloadSize; + remainingNaluLength -= curPayloadSize; + } + } + +CleanUp: + if (STATUS_FAILED(retStatus) && sizeCalculationOnly) { + payloadLength = 0; + payloadSubLenSize = 0; + } + + if (filledLength != NULL && filledSubLenSize != NULL) { + *filledLength = payloadLength; + *filledSubLenSize = payloadSubLenSize; + } + + LEAVES(); + return retStatus; +} + +STATUS depayH265FromRtpPayload(PBYTE pRawPacket, UINT32 packetLength, PBYTE pNaluData, PUINT32 pNaluLength, PBOOL pIsStart) +{ + ENTERS(); + STATUS retStatus = STATUS_SUCCESS; + UINT32 naluLength = packetLength, headerSize = 0; + UINT8 payloadHeaderType; + BOOL sizeCalculationOnly = (pNaluData == NULL); + BOOL isStartingPacket = TRUE; + PBYTE pCurPtrInNalu = pNaluData; + static BYTE start4ByteCode[] = {0x00, 0x00, 0x00, 0x01}; + + CHK(pRawPacket != NULL && pNaluLength != NULL, STATUS_NULL_ARG); + CHK(packetLength > 0, retStatus); + + payloadHeaderType = (pRawPacket[0] >> 1) & 0x3F; + + if (payloadHeaderType == H265_FU_TYPE_ID) { + isStartingPacket = (pRawPacket[2] & 0x80) != 0; + headerSize = H265_FU_HEADER_SIZE; + naluLength -= headerSize; + + if (isStartingPacket) { + naluLength += 2; + } + } + + if (isStartingPacket) { + naluLength += SIZEOF(start4ByteCode); + } + + CHK(!sizeCalculationOnly, retStatus); + CHK(naluLength <= *pNaluLength, STATUS_BUFFER_TOO_SMALL); + + if (isStartingPacket) { + MEMCPY(pCurPtrInNalu, start4ByteCode, SIZEOF(start4ByteCode)); + if (payloadHeaderType == H265_FU_TYPE_ID) { + pCurPtrInNalu[4] = ((pRawPacket[2] & 0x3F) << 1) | (pRawPacket[0] & 0x81); + pCurPtrInNalu[5] = pRawPacket[1]; + pCurPtrInNalu += 2; + } + pCurPtrInNalu += SIZEOF(start4ByteCode); + } + MEMCPY(pCurPtrInNalu, pRawPacket + headerSize, packetLength - headerSize); + +CleanUp: + if (STATUS_FAILED(retStatus) && sizeCalculationOnly) { + naluLength = 0; + } + + if (pNaluLength != NULL) { + *pNaluLength = naluLength; + } + + if (pIsStart != NULL) { + *pIsStart = isStartingPacket; + } + + LEAVES(); + return retStatus; +} diff --git a/src/source/Rtp/Codecs/RtpH265Payloader.h b/src/source/Rtp/Codecs/RtpH265Payloader.h new file mode 100644 index 0000000000..cc17d634ad --- /dev/null +++ b/src/source/Rtp/Codecs/RtpH265Payloader.h @@ -0,0 +1,41 @@ +/******************************************* +H265 RTP Payloader include file +*******************************************/ +#ifndef __KINESIS_VIDEO_WEBRTC_CLIENT_RTPH265PAYLOADER_H +#define __KINESIS_VIDEO_WEBRTC_CLIENT_RTPH265PAYLOADER_H + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#define H265_FU_HEADER_SIZE 3 +#define H265_FU_TYPE_ID 49 + +// https://www.rfc-editor.org/rfc/rfc7798.html#section-4.4.3 + +/* + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | PayloadHdr (Type=49) | FU header | DONL (cond) | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-| + * | DONL (cond) | | + * |-+-+-+-+-+-+-+-+ | + * | FU payload | + * | | + * | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | :...OPTIONAL RTP padding | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ + +STATUS createPayloadForH265(UINT32, PBYTE, UINT32, PBYTE, PUINT32, PUINT32, PUINT32); +STATUS getNextNaluLengthH265(PBYTE, UINT32, PUINT32, PUINT32); +STATUS createPayloadFromNaluH265(UINT32, PBYTE, UINT32, PPayloadArray, PUINT32, PUINT32); +STATUS depayH265FromRtpPayload(PBYTE, UINT32, PBYTE, PUINT32, PBOOL); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/tst/RtpFunctionalityTest.cpp b/tst/RtpFunctionalityTest.cpp index 387aa5fb0b..7a114035c9 100644 --- a/tst/RtpFunctionalityTest.cpp +++ b/tst/RtpFunctionalityTest.cpp @@ -7,6 +7,7 @@ namespace video { namespace webrtcclient { #define NUMBER_OF_FRAME_FILES 403 +#define NUMBER_OF_H265_FRAME_FILES 1500 #define DEFAULT_FPS_VALUE 25 BYTE start4ByteCode[] = {0x00, 0x00, 0x00, 0x01}; @@ -108,7 +109,7 @@ TEST_F(RtpFunctionalityTest, marshallUnmarshallH264Data) } fileIndex = fileIndex % NUMBER_OF_FRAME_FILES + 1; - EXPECT_EQ(STATUS_SUCCESS, readFrameData((PBYTE) payload, (PUINT32) &payloadLen, fileIndex, (PCHAR) "../samples/h264SampleFrames")); + EXPECT_EQ(STATUS_SUCCESS, readFrameData((PBYTE) payload, (PUINT32) &payloadLen, fileIndex, (PCHAR) "../samples/h264SampleFrames", RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE)); // First call for payload size and sub payload length size EXPECT_EQ(STATUS_SUCCESS, @@ -187,7 +188,7 @@ TEST_F(RtpFunctionalityTest, packingUnpackingVerifySameH264Frame) payloadArray.payloadSubLength = NULL; for (fileIndex = 1; fileIndex <= NUMBER_OF_FRAME_FILES; fileIndex++) { - EXPECT_EQ(STATUS_SUCCESS, readFrameData((PBYTE) payload, (PUINT32) &payloadLen, fileIndex, (PCHAR) "../samples/h264SampleFrames")); + EXPECT_EQ(STATUS_SUCCESS, readFrameData((PBYTE) payload, (PUINT32) &payloadLen, fileIndex, (PCHAR) "../samples/h264SampleFrames", RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE)); // First call for payload size and sub payload length size EXPECT_EQ(STATUS_SUCCESS, @@ -261,6 +262,103 @@ TEST_F(RtpFunctionalityTest, packingUnpackingVerifySameH264Frame) MEMFREE(depayload); } +TEST_F(RtpFunctionalityTest, packingUnpackingVerifySameH265Frame) +{ + PBYTE payload = (PBYTE) MEMCALLOC(1, 200000); // Assuming this is enough + PBYTE depayload = (PBYTE) MEMCALLOC(1, 1500); // This is more than max mtu + UINT32 depayloadSize = 1500; + UINT32 payloadLen = 0; + UINT32 fileIndex = 0; + PayloadArray payloadArray; + UINT32 i = 0; + UINT32 offset = 0; + UINT32 newPayloadLen = 0, newPayloadSubLen = 0; + BOOL isStartPacket = FALSE; + PBYTE pCurPtrInPayload = NULL; + UINT32 remainPayloadLen = 0; + UINT32 startIndex = 0, naluLength = 0; + UINT32 startLen = 0; + + payloadArray.maxPayloadLength = 0; + payloadArray.maxPayloadSubLenSize = 0; + payloadArray.payloadBuffer = NULL; + payloadArray.payloadSubLength = NULL; + + for (fileIndex = 1; fileIndex <= NUMBER_OF_H265_FRAME_FILES; fileIndex++) { + EXPECT_EQ(STATUS_SUCCESS, readFrameData((PBYTE) payload, (PUINT32) &payloadLen, fileIndex, (PCHAR) "../samples/h265SampleFrames", RTC_CODEC_H265)); + + // First call for payload size and sub payload length size + EXPECT_EQ(STATUS_SUCCESS, + createPayloadForH265(DEFAULT_MTU_SIZE_BYTES, (PBYTE) payload, payloadLen, NULL, &payloadArray.payloadLength, NULL, + &payloadArray.payloadSubLenSize)); + + if (payloadArray.payloadLength > payloadArray.maxPayloadLength) { + if (payloadArray.payloadBuffer != NULL) { + MEMFREE(payloadArray.payloadBuffer); + } + payloadArray.payloadBuffer = (PBYTE) MEMALLOC(payloadArray.payloadLength); + payloadArray.maxPayloadLength = payloadArray.payloadLength; + } + if (payloadArray.payloadSubLenSize > payloadArray.maxPayloadSubLenSize) { + if (payloadArray.payloadSubLength != NULL) { + MEMFREE(payloadArray.payloadSubLength); + } + payloadArray.payloadSubLength = (PUINT32) MEMALLOC(payloadArray.payloadSubLenSize * SIZEOF(UINT32)); + payloadArray.maxPayloadSubLenSize = payloadArray.payloadSubLenSize; + } + + // Second call with actual buffer to fill in data + EXPECT_EQ(STATUS_SUCCESS, + createPayloadForH265(DEFAULT_MTU_SIZE_BYTES, (PBYTE) payload, payloadLen, payloadArray.payloadBuffer, &payloadArray.payloadLength, + payloadArray.payloadSubLength, &payloadArray.payloadSubLenSize)); + + EXPECT_LT(0, payloadArray.payloadSubLenSize); + + offset = 0; + + for (i = 0; i < payloadArray.payloadSubLenSize; i++) { + EXPECT_EQ(STATUS_SUCCESS, + depayH265FromRtpPayload(payloadArray.payloadBuffer + offset, payloadArray.payloadSubLength[i], NULL, &newPayloadSubLen, + &isStartPacket)); + newPayloadLen += newPayloadSubLen; + if (isStartPacket) { + newPayloadLen -= SIZEOF(start4ByteCode); + } + EXPECT_LT(0, newPayloadSubLen); + offset += payloadArray.payloadSubLength[i]; + } + EXPECT_LE(newPayloadLen, payloadLen); + + offset = 0; + newPayloadLen = 0; + isStartPacket = FALSE; + pCurPtrInPayload = payload; + remainPayloadLen = payloadLen; + for (i = 0; i < payloadArray.payloadSubLenSize; i++) { + newPayloadSubLen = depayloadSize; + EXPECT_EQ(STATUS_SUCCESS, + depayH265FromRtpPayload(payloadArray.payloadBuffer + offset, payloadArray.payloadSubLength[i], depayload, &newPayloadSubLen, + &isStartPacket)); + if (isStartPacket) { + EXPECT_EQ(STATUS_SUCCESS, getNextNaluLengthH265(pCurPtrInPayload, remainPayloadLen, &startIndex, &naluLength)); + pCurPtrInPayload += startIndex; + startLen = SIZEOF(start4ByteCode); + } else { + startLen = 0; + } + EXPECT_TRUE(MEMCMP(pCurPtrInPayload, depayload + startLen, newPayloadSubLen - startLen) == 0); + pCurPtrInPayload += newPayloadSubLen - startLen; + remainPayloadLen -= newPayloadSubLen; + offset += payloadArray.payloadSubLength[i]; + } + } + + MEMFREE(payloadArray.payloadBuffer); + MEMFREE(payloadArray.payloadSubLength); + MEMFREE(payload); + MEMFREE(depayload); +} + TEST_F(RtpFunctionalityTest, packingUnpackingVerifySameOpusFrame) { BYTE payload[] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05}; @@ -452,15 +550,26 @@ TEST_F(RtpFunctionalityTest, invalidNaluParse) UINT32 startIndex = 0, naluLength = 0; EXPECT_EQ(STATUS_RTP_INVALID_NALU, getNextNaluLength(data, 3, &startIndex, &naluLength)); EXPECT_EQ(STATUS_RTP_INVALID_NALU, getNextNaluLength(data1, 7, &startIndex, &naluLength)); + + EXPECT_EQ(STATUS_RTP_INVALID_NALU, getNextNaluLengthH265(data, 3, &startIndex, &naluLength)); + EXPECT_EQ(STATUS_RTP_INVALID_NALU, getNextNaluLengthH265(data1, 7, &startIndex, &naluLength)); } TEST_F(RtpFunctionalityTest, validNaluParse) { BYTE data[] = {0x00, 0x00, 0x00, 0x01, 0x00, 0x02}; UINT32 startIndex = 0, naluLength = 0; + EXPECT_EQ(STATUS_SUCCESS, getNextNaluLength(data, 6, &startIndex, &naluLength)); EXPECT_EQ(4, startIndex); EXPECT_EQ(2, naluLength); + + startIndex = 0; + naluLength = 0; + + EXPECT_EQ(STATUS_SUCCESS, getNextNaluLengthH265(data, 6, &startIndex, &naluLength)); + EXPECT_EQ(4, startIndex); + EXPECT_EQ(2, naluLength); } TEST_F(RtpFunctionalityTest, validMultipleNaluParse) @@ -540,4 +649,4 @@ TEST_F(RtpFunctionalityTest, freeTransceiverApiTest) } // namespace video } // namespace kinesis } // namespace amazonaws -} // namespace com \ No newline at end of file +} // namespace com diff --git a/tst/SdpApiTest.cpp b/tst/SdpApiTest.cpp index fca09c031c..676f4e13c7 100644 --- a/tst/SdpApiTest.cpp +++ b/tst/SdpApiTest.cpp @@ -350,6 +350,36 @@ TEST_F(SdpApiTest, setTransceiverPayloadTypes_HasRtxType) doubleListFree(pTransceivers); } +TEST_F(SdpApiTest, setTransceiverPayloadTypes_HasRtxType_H265) +{ + PHashTable pCodecTable; + PHashTable pRtxTable; + PDoubleList pTransceivers; + KvsRtpTransceiver transceiver; + transceiver.sender.track.codec = RTC_CODEC_H265; + transceiver.transceiver.direction = RTC_RTP_TRANSCEIVER_DIRECTION_SENDRECV; + transceiver.sender.packetBuffer = NULL; + transceiver.sender.retransmitter = NULL; + transceiver.rollingBufferDurationSec = DEFAULT_ROLLING_BUFFER_DURATION_IN_SECONDS; + transceiver.rollingBufferBitratebps = DEFAULT_EXPECTED_VIDEO_BIT_RATE; + EXPECT_EQ(STATUS_SUCCESS, hashTableCreate(&pCodecTable)); + EXPECT_EQ(STATUS_SUCCESS, hashTablePut(pCodecTable, RTC_CODEC_H265, 1)); + EXPECT_EQ(STATUS_SUCCESS, hashTableCreate(&pRtxTable)); + EXPECT_EQ(STATUS_SUCCESS, hashTablePut(pRtxTable, RTC_CODEC_H265, 2)); + EXPECT_EQ(STATUS_SUCCESS, doubleListCreate(&pTransceivers)); + EXPECT_EQ(STATUS_SUCCESS, doubleListInsertItemHead(pTransceivers, (UINT64)(&transceiver))); + EXPECT_EQ(STATUS_SUCCESS, setTransceiverPayloadTypes(pCodecTable, pRtxTable, pTransceivers)); + EXPECT_EQ(1, transceiver.sender.payloadType); + EXPECT_EQ(2, transceiver.sender.rtxPayloadType); + EXPECT_NE((PRtpRollingBuffer) NULL, transceiver.sender.packetBuffer); + EXPECT_NE((PRetransmitter) NULL, transceiver.sender.retransmitter); + hashTableFree(pCodecTable); + hashTableFree(pRtxTable); + freeRtpRollingBuffer(&transceiver.sender.packetBuffer); + freeRetransmitter(&transceiver.sender.retransmitter); + doubleListFree(pTransceivers); +} + TEST_F(SdpApiTest, populateSingleMediaSection_TestTxSendRecv) { PRtcPeerConnection offerPc = NULL; @@ -451,6 +481,37 @@ TEST_F(SdpApiTest, populateSingleMediaSection_TestTxSendOnly) freePeerConnection(&offerPc); } +TEST_F(SdpApiTest, populateSingleMediaSection_TestTxSendOnly_H265) +{ + PRtcPeerConnection offerPc = NULL; + RtcConfiguration configuration; + RtcSessionDescriptionInit sessionDescriptionInit; + + MEMSET(&configuration, 0x00, SIZEOF(RtcConfiguration)); + + // Create peer connection + EXPECT_EQ(createPeerConnection(&configuration, &offerPc), STATUS_SUCCESS); + + RtcMediaStreamTrack track; + PRtcRtpTransceiver pTransceiver; + RtcRtpTransceiverInit rtcRtpTransceiverInit; + rtcRtpTransceiverInit.direction = RTC_RTP_TRANSCEIVER_DIRECTION_SENDONLY; + + MEMSET(&track, 0x00, SIZEOF(RtcMediaStreamTrack)); + + track.kind = MEDIA_STREAM_TRACK_KIND_VIDEO; + track.codec = RTC_CODEC_H265; + STRCPY(track.streamId, "myKvsVideoStream"); + STRCPY(track.trackId, "myTrack"); + + EXPECT_EQ(STATUS_SUCCESS, addTransceiver(offerPc, &track, &rtcRtpTransceiverInit, &pTransceiver)); + EXPECT_EQ(STATUS_SUCCESS, createOffer(offerPc, &sessionDescriptionInit)); + EXPECT_PRED_FORMAT2(testing::IsSubstring, "sendonly", sessionDescriptionInit.sdp); + + closePeerConnection(offerPc); + freePeerConnection(&offerPc); +} + TEST_F(SdpApiTest, populateSingleMediaSection_TestTxRecvOnly) { PRtcPeerConnection offerPc = NULL; diff --git a/tst/WebRTCClientTestFixture.h b/tst/WebRTCClientTestFixture.h index 5e372b501a..4c9d7f7642 100644 --- a/tst/WebRTCClientTestFixture.h +++ b/tst/WebRTCClientTestFixture.h @@ -273,7 +273,7 @@ class WebRtcClientTestBase : public ::testing::Test { return getExponentialBackoffRetryStrategyWaitTime(pKvsRetryStrategy, retryWaitTime); } - STATUS readFrameData(PBYTE pFrame, PUINT32 pSize, UINT32 index, PCHAR frameFilePath) + STATUS readFrameData(PBYTE pFrame, PUINT32 pSize, UINT32 index, PCHAR frameFilePath, RTC_CODEC rtcCodec) { STATUS retStatus = STATUS_SUCCESS; CHAR filePath[MAX_PATH_LEN + 1]; @@ -281,7 +281,16 @@ class WebRtcClientTestBase : public ::testing::Test { CHK(pFrame != NULL && pSize != NULL, STATUS_NULL_ARG); - SNPRINTF(filePath, MAX_PATH_LEN, "%s/frame-%04d.h264", frameFilePath, index); + switch (rtcCodec) { + case RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE: + SNPRINTF(filePath, MAX_PATH_LEN, "%s/frame-%04d.h264", frameFilePath, index); + break; + case RTC_CODEC_H265: + SNPRINTF(filePath, MAX_PATH_LEN, "%s/frame-%04d.h265", frameFilePath, index); + break; + default: + break; + } // Get the size and read into frame CHK_STATUS(readFile(filePath, TRUE, NULL, &size));