Skip to content

Commit

Permalink
Release 1.10.2 (#1981)
Browse files Browse the repository at this point in the history 
* Add H265 and AAC frames (#1962)

* aac and h265 frames

* update readme

* update cmake with h265 and aac

* clang formay

* [Sample] GStreamer media receiver (#1963)

* create the gst receiver

* fix ci

* cmake flag

* clang format, add cmake flag in ci

* install gstreamer on ci

* use macos12

* use defined values

* indent

* clang-format

* readme and commentsa

* clang :(

* install glib to fix ci

* -DCOMPILER_WARNINGS=FALSE to fix the CI

* fix macos version for clang

* address comments and cleanup

* new viewer with gst

* clang-format

* missing null check

* address comments

* -DCOMPILER_WARNINGS=TRUE

* clang-format

* gst_init_check

* fix CI

* fix CI

* [Bug] GStreamer sample (#1972)

* gst-sample-fix

* modify the message

* fix windows

* set to null

* fix pts and eos in sample

* unused var removed

* move null init

* Version check job (#1974)

* Version check job

* Working Version check template

* Retest with dev

* Version lint

* Version lint

* use new frames (#1978)

* Sample TWCC implementation (#1957)

* encoder bitrate change based on twcc

* Change to 5% inc and dec

* modify percentages

* ema based calc

* EMa fix

* Nits

* Readme

* flip

* memset remove

* Readme update, move enable flags to createSampleConfiguration

* Add codecov token

* [Feature] Add H265 support with SDP, RTP payloader-deplayloader (#1965)

* Add RtpH265Payloader.c and RtpH265Payloader.h

* Add support for H265 in PeerConnection/SessionDescription

* Add support for H265 in PeerConnection/PeerConnection

* Add support for H265 in PeerConnection/Rtp

* Add support for H265 in samples/Common.c

* Add support for H265 in samples/kvsWebRTCClientMaster.c

* rtp, sdp fix, flag removed, clang fixed, windows build fixed, new test added

* test fix

* cleanup

* cleanup

* remove #if 0

* clang

* presentation ts fix

* clang-format fix

* PKG_CONFIG_PATH in kvscommon

* missing bracket

* fix all builds

* ci

* cleanup

* fix windows build, rename h265 defs

* remove duplicate line from h264 and h265

* sample changes

* address comments

* clang-format

* gst sample

* cleanup args

* clang-format

* cleanup

* add sdp tests

* address comments

* address commentas

* set default payload type only once

* address comments

* fix height and width

* sdp change

---------

Co-authored-by: Hongli Wang <hongliwo@amazon.com>

* Version bump, fix tests, remove enableIceStats

* Remove AAC related changes

* Remove AAC frames

* Clang format

* fix kvsWebrtcClientMasterGstSample.c name in CMakeLists

---------

Co-authored-by: Divya Sampath Kumar <disa6302@colorado.edu>
Co-authored-by: Hongli Wang <hongliwo@amazon.com>
  • Loading branch information
@niyatim23 @disa6302 @hongliwo
3 people committed May 9, 2024
1 parent f107343 commit 4f1d3ad
Show file tree
Hide file tree
Showing 1,527 changed files with 1,558 additions and 203 deletions.
9 changes: 7 additions & 2 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ on:
- master
jobs:
clang-format-check:
runs-on: macos-latest
runs-on: macos-12
steps:
- name: Clone repository
uses: actions/checkout@v3
Expand Down Expand Up @@ -78,6 +78,7 @@ jobs:
runs-on: macos-13-xlarge
env:
AWS_KVS_LOG_LEVEL: 2
PKG_CONFIG_PATH: /usr/local/opt/pkgconfig
permissions:
id-token: write
contents: read
Expand All @@ -89,6 +90,9 @@ jobs:
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: ${{ secrets.AWS_REGION }}
- name: Install dependencies
run: |
brew install gstreamer glib
- name: Build repository
run: |
brew unlink openssl
Expand Down Expand Up @@ -552,6 +556,7 @@ jobs:
- name: Install dependencies
shell: powershell
run: |
choco install pkgconfiglite
choco install gstreamer --version=1.16.3
choco install gstreamer-devel --version=1.16.3
curl.exe -o C:\tools\pthreads-w32-2-9-1-release.zip ftp://sourceware.org/pub/pthreads-win32/pthreads-w32-2-9-1-release.zip
Expand Down Expand Up @@ -580,7 +585,7 @@ jobs:
shell: powershell
run: |
$env:Path += ';C:\webrtc\open-source\bin;C:\tools\pthreads-w32-2-9-1-release\Pre-built.2\dll\x64;C:\webrtc\build'
& "C:\webrtc\build\tst\webrtc_client_test.exe" --gtest_filter="-SignalingApiFunctionalityTest.receivingIceConfigOffer_SlowClockSkew:SignalingApiFunctionalityTest.iceServerConfigRefreshConnectedAuthExpiration:SignalingApiFunctionalityTest.receivingIceConfigOffer_FastClockSkew:SignalingApiFunctionalityTest.receivingIceConfigOffer_FastClockSkew_VerifyOffsetRemovedWhenClockFixed:DataChannelFunctionalityTest.*:DtlsApiTest.*:IceApiTest.*:IceFunctionalityTest.*:PeerConnectionFunctionalityTest.*:TurnConnectionFunctionalityTest.*:RtpFunctionalityTest.marshallUnmarshallH264Data:RtpFunctionalityTest.packingUnpackingVerifySameH264Frame:RtcpFunctionalityTest.onRtcpPacketCompound:RtcpFunctionalityTest.twcc3"
& "C:\webrtc\build\tst\webrtc_client_test.exe" --gtest_filter="-SignalingApiFunctionalityTest.receivingIceConfigOffer_SlowClockSkew:SignalingApiFunctionalityTest.iceServerConfigRefreshConnectedAuthExpiration:SignalingApiFunctionalityTest.receivingIceConfigOffer_FastClockSkew:SignalingApiFunctionalityTest.receivingIceConfigOffer_FastClockSkew_VerifyOffsetRemovedWhenClockFixed:DataChannelFunctionalityTest.*:DtlsApiTest.*:IceApiTest.*:IceFunctionalityTest.*:PeerConnectionFunctionalityTest.*:TurnConnectionFunctionalityTest.*:RtpFunctionalityTest.marshallUnmarshallH264Data:RtpFunctionalityTest.packingUnpackingVerifySameH264Frame:RtpFunctionalityTest.packingUnpackingVerifySameH265Frame:RtcpFunctionalityTest.onRtcpPacketCompound:RtcpFunctionalityTest.twcc3"
# windows-msvc-mbedtls:
# runs-on: windows-2022
# env:
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/codecov.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ jobs:
runs-on: ubuntu-20.04
env:
AWS_KVS_LOG_LEVEL: 2
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
permissions:
id-token: write
contents: read
Expand Down
42 changes: 42 additions & 0 deletions .github/workflows/version-check.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
name: Check Version Mismatch between PR branch and master.

on:
pull_request:
branches:
- master

jobs:
check-version:
runs-on: ubuntu-latest
steps:
- name: Checkout PR branch
uses: actions/checkout@v4

- name: Get version from PR
id: pr_version
run: |
PR_VERSION=$(grep -Po 'KinesisVideoWebRTCClient VERSION \K[0-9]+\.[0-9]+\.[0-9]+' CMakeLists.txt)
echo "PR_VERSION=$PR_VERSION" >> "$GITHUB_ENV"
echo "PR Version: $PR_VERSION"
- name: Checkout master branch
uses: actions/checkout@v4
with:
ref: master

- name: Get version from master
id: master_version
run: |
MASTER_VERSION=$(grep -Po 'KinesisVideoWebRTCClient VERSION \K[0-9]+\.[0-9]+\.[0-9]+' CMakeLists.txt)
echo "MASTER_VERSION=$MASTER_VERSION" >> "$GITHUB_ENV"
echo "Master version: $MASTER_VERSION"
- name: Compare versions
run: |
echo "Comparing PR Version: $PR_VERSION with Master Version: $MASTER_VERSION"
if [ "$MASTER_VERSION" == "$PR_VERSION" ]; then
echo "Please update the version in CMakeLists.txt file (project(KinesisVideoWebRTCClient VERSION <ver-string> LANGUAGES C). Any PR getting merged to master requires a version update"
exit 1
else
echo "Version update detected."
fi
3 changes: 2 additions & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ include(CheckIncludeFiles)
include(CheckFunctionExists)

# The version MUST be updated before every release
project(KinesisVideoWebRTCClient VERSION 1.10.1 LANGUAGES C)
project(KinesisVideoWebRTCClient VERSION 1.10.2 LANGUAGES C)

# User Flags
option(ADD_MUCLIBC "Add -muclibc c flag" OFF)
Expand Down Expand Up @@ -440,6 +440,7 @@ if (BUILD_SAMPLE)
# copy sample frames to build folder, in case developer runs sample program with command `samples/kvsWebrtcClientMaster` from `build` dir.
file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/samples/opusSampleFrames" DESTINATION .)
file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/samples/h264SampleFrames" DESTINATION .)
file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/samples/h265SampleFrames" DESTINATION .)

add_subdirectory(samples)
endif()
Expand Down
112 changes: 108 additions & 4 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,15 +268,18 @@ After executing `make` you will have sample applications in your `build/samples`
#### Sample: kvsWebrtcClientMaster
This application sends sample H264/Opus frames (path: `/samples/h264SampleFrames` and `/samples/opusSampleFrames`) via WebRTC. It also accepts incoming audio, if enabled in the browser. When checked in the browser, it prints the metadata of the received audio packets in your terminal. To run:
```shell
./samples/kvsWebrtcClientMaster <channelName>
./samples/kvsWebrtcClientMaster <channelName> <storage-option> <audio-codec> <video-codec>
```

To use the **Storage for WebRTC** feature, run the same command as above but with an additional command line arg to enable the feature.

```shell
./samples/kvsWebrtcClientMaster <channelName> 1
./samples/kvsWebrtcClientMaster <channelName> 1 <audio-codec> <video-codec>
```

Allowed audio-codec: opus (default codec if nothing is specified)
Allowed video-codec: h264 (default codec if nothing is specified), h265

#### Sample: kvsWebrtcClientMasterGstSample
This application can send media from a GStreamer pipeline using test H264/Opus frames, device `autovideosrc` and `autoaudiosrc` input, or a received RTSP stream. It also will playback incoming audio via an `autoaudiosink`. To run:
```shell
Expand All @@ -288,11 +291,59 @@ Pass the desired media and source type when running the sample. The mediaType ca
./samples/kvsWebrtcClientMasterGstSample <channelName> <mediaType> rtspsrc rtsp://<rtspUri>
```

Using the testsrc with audio and video-codec
```shell
./samples/kvsWebrtcClientMasterGstSample <channelName> <mediaType> <sourceType> <audio-codec> <video-codec>
```

Example:
```shell
./samples/kvsWebrtcClientMasterGstSample <channelName> audio-video testsrc opus h264
```

Allowed audio-codec: opus (default codec if nothing is specified)
Allowed video-codec: h264 (default codec if nothing is specified), h265

#### Sample: kvsWebrtcClientViewer
This application accepts sample H264/Opus frames and prints them out. To run:
This application accepts sample H264/Opus frames by default. You can use other supported codecs by changing the value for `videoTrack.codec` and `audioTrack.codec` in _Common.c_. By default, this sample only logs the size of the audio and video buffer it receives. To write these frames to a file using GStreamer, use the _kvsWebrtcClientViewerGstSample_ instead.

To run:
```shell
./samples/kvsWebrtcClientViewer <channelName> <audio-codec> <video-codec>
```

Allowed audio-codec: opus (default codec if nothing is specified)
Allowed video-codec: h264 (default codec if nothing is specified), h265

#### Sample: kvsWebrtcClientViewerGstSample
This application is similar to the kvsWebrtcClientViewer. However, instead of just logging the media it receives, it generates a file using filesink. Make sure that your device has enough space to write the media to a file. You can also customize the receiving logic by modifying the functions in _GstAudioVideoReceiver.c_

To run:
```shell
./samples/kvsWebrtcClientViewer <channelName>
./samples/kvsWebrtcClientViewerGstSample <channelName> <mediaType> <audio-codec> <video-codec>
```

Allowed audio-codec: opus (default codec if nothing is specified)
Allowed video-codec: h264 (default codec if nothing is specified), h265

##### Known issues:
Our GStreamer samples leverage [MatroskaMux](https://gstreamer.freedesktop.org/documentation/matroska/matroskamux.html?gi-language=c) to receive media from its peer and save it to a file. However, MatroskaMux is designed for scenarios where the media's format remains constant throughout streaming. When the media's format changes mid-streaming (referred to as "caps changes"), MatroskaMux encounters limitations, its behavior cannot be predicted and it may be unable to handle these changes, resulting in an error message like:

```shell
matroskamux matroska-mux.c:1134:gst_matroska_mux_video_pad_setcaps:<mux> error: Caps changes are not supported by Matroska
```
To address this issue, users need to adapt the pipeline to utilize components capable of managing dynamic changes in media formats. This might involve integrating different muxers or customizing the pipeline to handle caps changes effectively.

#### Sample: Generating sample frames

##### H264
```shell
gst-launch-1.0 videotestsrc pattern=ball num-buffers=1500 ! timeoverlay ! videoconvert ! video/x-raw,format=I420,width=1280,height=720,framerate=25/1 ! queue ! x264enc bframes=0 speed-preset=veryfast bitrate=512 byte-stream=TRUE tune=zerolatency ! video/x-h264,stream-format=byte-stream,alignment=au,profile=baseline ! multifilesink location="frame-%04d.h264" index=1
```

##### H265
```shell
gst-launch-1.0 videotestsrc pattern=ball num-buffers=1500 ! timeoverlay ! videoconvert ! video/x-raw,format=I420,width=1280,height=720,framerate=25/1 ! queue ! x265enc speed-preset=veryfast bitrate=512 tune=zerolatency ! video/x-h265,stream-format=byte-stream,alignment=au,profile=main ! multifilesink location="frame-%04d.h265" index=1
```

### Viewing Master Samples
Expand Down Expand Up @@ -350,6 +401,33 @@ createLwsIotCredentialProvider(
freeIotCredentialProvider(&pSampleConfiguration->pCredentialProvider);
```
## TWCC support
Transport Wide Congestion Control (TWCC) is a mechanism in WebRTC designed to enhance the performance and reliability of real-time communication over the internet. TWCC addresses the challenges of network congestion by providing detailed feedback on the transport of packets across the network, enabling adaptive bitrate control and optimization of media streams in real-time. This feedback mechanism is crucial for maintaining high-quality audio and video communication, as it allows senders to adjust their transmission strategies based on comprehensive information about packet losses, delays, and jitter experienced across the entire transport path.
The importance of TWCC in WebRTC lies in its ability to ensure efficient use of available network bandwidth while minimizing the negative impacts of network congestion. By monitoring the delivery of packets across the network, TWCC helps identify bottlenecks and adjust the media transmission rates accordingly. This dynamic approach to congestion control is essential for preventing degradation in call quality, such as pixelation, stuttering, or drops in audio and video streams, especially in environments with fluctuating network conditions.
To learn more about TWCC, check [TWCC spec](https://datatracker.ietf.org/doc/html/draft-holmer-rmcat-transport-wide-cc-extensions-01)
### Enabling TWCC support
TWCC is enabled by default in the SDK samples (via `pSampleConfiguration->enableTwcc`) flag. In order to disable it, set this flag to `FALSE`.
```c
pSampleConfiguration->enableTwcc = FALSE;
```

If not using the samples directly, 2 things need to be done to set up Twcc:
1. Set the `disableSenderSideBandwidthEstimation` to `FALSE`:
```c
configuration.kvsRtcConfiguration.disableSenderSideBandwidthEstimation = FALSE;
```
2. Set the callback that will have the business logic to modify the bitrate based on packet loss information. The callback can be set using `peerConnectionOnSenderBandwidthEstimation()`:
```c
CHK_STATUS(peerConnectionOnSenderBandwidthEstimation(pSampleStreamingSession->pPeerConnection, (UINT64) pSampleStreamingSession,
sampleSenderBandwidthEstimationHandler));
```
## Use Pre-generated Certificates
The certificate generating function ([createCertificateAndKey](https://awslabs.github.io/amazon-kinesis-video-streams-webrtc-sdk-c/Dtls__openssl_8c.html#a451c48525b0c0a8919a880d6834c1f7f)) in createDtlsSession() can take between 5 - 15 seconds in low performance embedded devices, it is called for every peer connection creation when KVS WebRTC receives an offer. To avoid this extra start-up latency, certificate can be pre-generated and passed in when offer comes.
Expand Down Expand Up @@ -434,6 +512,32 @@ To disable threadpool, run `cmake .. -DENABLE_KVS_THREADPOOL=OFF`

Starting version 1.10.0, threadpool usage provides latency improvements in connection establishment. Note, that increasing the number of minimum threads can increase stack memory usage. So, ensure to increase with caution.

### Set up TWCC
TWCC is a mechanism in WebRTC designed to enhance the performance and reliability of real-time communication over the Internet. TWCC addresses the challenges of network congestion by providing detailed feedback on the transport of packets across the network, enabling adaptive bitrate control and optimization of
media streams in real-time. This feedback mechanism is crucial for maintaining high-quality audio and video communication, as it allows senders to adjust their transmission strategies based on comprehensive information about packet losses, delays, and jitter experienced across the entire transport path.
The importance of TWCC in WebRTC lies in its ability to ensure efficient use of available network bandwidth while minimizing the negative impacts of network congestion. By monitoring the delivery of packets across the network, TWCC helps identify bottlenecks and adjust the media transmission rates accordingly.
This dynamic approach to congestion control is essential for preventing degradation in call quality, such as pixelation, stuttering, or drops in audio and video streams, especially in environments with fluctuating network conditions. To learn more about TWCC, you can refer to the [RFC draft](https://datatracker.ietf.org/doc/html/draft-holmer-rmcat-transport-wide-cc-extensions-01)

In order to enable TWCC usage in the SDK, 2 things need to be set up:

1. Set the `disableSenderSideBandwidthEstimation` to FALSE. In our samples, the value is set using `enableTwcc` flag in `pSampleConfiguration`

```c
pSampleConfiguration->enableTwcc = TRUE; // to enable TWCC
pSampleConfiguration->enableTwcc = FALSE; // to disable TWCC
configuration.kvsRtcConfiguration.disableSenderSideBandwidthEstimation = !pSampleConfiguration->enableTwcc;
```

2. Set the callback that will have the business logic to modify the bitrate based on packet loss information. The callback can be set using `peerConnectionOnSenderBandwidthEstimation()`.

```c
CHK_STATUS(peerConnectionOnSenderBandwidthEstimation(pSampleStreamingSession->pPeerConnection, (UINT64) pSampleStreamingSession,
sampleSenderBandwidthEstimationHandler));
```
By default, our SDK enables TWCC listener. The SDK has a sample implementation to integrate TWCC into the Gstreamer pipeline via the `sampleSenderBandwidthEstimationHandler` callback. To get more details, look for this specific callback.
### Setting ICE related timeouts
There are some default timeout values set for different steps in ICE in the [KvsRtcConfiguration](https://awslabs.github.io/amazon-kinesis-video-streams-webrtc-sdk-c/structKvsRtcConfiguration.html). These are configurable in the application. While the defaults are generous, there could be applications that might need more flexibility to improve chances of connection establishment because of poor network.
Expand Down
14 changes: 14 additions & 0 deletions samples/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ link_directories(${OPEN_SRC_INSTALL_PREFIX}/lib)
# copy sample frames to this subproject build folder, in case developer runs sample program with command `kvsWebrtcClientMaster` from `build/samples` dir.
file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/opusSampleFrames" DESTINATION .)
file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/h264SampleFrames" DESTINATION .)
file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/h265SampleFrames" DESTINATION .)

add_executable(
kvsWebrtcClientMaster
Expand All @@ -78,13 +79,26 @@ if(GST_FOUND)
add_executable(
kvsWebrtcClientMasterGstSample
Common.c
GstAudioVideoReceiver.c
kvsWebrtcClientMasterGstSample.c
)
target_link_libraries(kvsWebrtcClientMasterGstSample kvsWebrtcClient kvsWebrtcSignalingClient ${EXTRA_DEPS} ${GST_SAMPLE_LIBRARIES} kvsCommonLws kvspicUtils websockets)

install(TARGETS kvsWebrtcClientMasterGstSample
RUNTIME DESTINATION bin
)

add_executable(
kvsWebrtcClientViewerGstSample
Common.c
GstAudioVideoReceiver.c
kvsWebRTCClientViewerGstSample.c
)
target_link_libraries(kvsWebrtcClientViewerGstSample kvsWebrtcClient kvsWebrtcSignalingClient ${EXTRA_DEPS} ${GST_SAMPLE_LIBRARIES} kvsCommonLws kvspicUtils websockets)

install(TARGETS kvsWebrtcClientViewerGstSample
RUNTIME DESTINATION bin
)
endif()

install(TARGETS kvsWebrtcClientMaster kvsWebrtcClientViewer discoverNatBehavior
Expand Down
Loading

0 comments on commit 4f1d3ad

Please sign in to comment.