Skip to content

Commit

Permalink
VP9 (#465)
Browse files Browse the repository at this point in the history 
* vp9

* working

* cleaning

* fix video transcoding

* fix track websocket

* disable vp9 tests for now
  • Loading branch information
@frostbyte73
frostbyte73 authored Aug 16, 2023
1 parent e056e81 commit cf060cc
Show file tree
Hide file tree
Showing 13 changed files with 258 additions and 196 deletions.
14 changes: 7 additions & 7 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ require (
github.com/go-logr/logr v1.2.4
github.com/googleapis/gax-go/v2 v2.11.0
github.com/gorilla/websocket v1.5.0
github.com/livekit/livekit-server v1.4.4-0.20230612120056-afa773374840
github.com/livekit/livekit-server v1.4.5-0.20230814182001-77c8e824735b
github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1
github.com/livekit/protocol v1.6.0
github.com/livekit/psrpc v0.3.2
github.com/livekit/server-sdk-go v1.0.16-0.20230811173715-bab67d4a7670
github.com/livekit/server-sdk-go v1.0.16-0.20230815025737-c12cd2eb8fe8
github.com/pion/rtp v1.8.1
github.com/pion/webrtc/v3 v3.2.14
github.com/pkg/errors v0.9.1
Expand Down Expand Up @@ -73,7 +73,7 @@ require (
github.com/klauspost/compress v1.16.5 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/lithammer/shortuuid/v4 v4.0.0 // indirect
github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a // indirect
github.com/livekit/mediatransportutil v0.0.0-20230814030822-8d5de0008b08 // indirect
github.com/mackerelio/go-osstat v0.2.4 // indirect
github.com/magefile/mage v1.15.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
Expand All @@ -85,7 +85,7 @@ require (
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pion/datachannel v1.5.5 // indirect
github.com/pion/dtls/v2 v2.2.7 // indirect
github.com/pion/ice/v2 v2.3.9 // indirect
github.com/pion/ice/v2 v2.3.10 // indirect
github.com/pion/interceptor v0.1.17 // indirect
github.com/pion/logging v0.2.2 // indirect
github.com/pion/mdns v0.0.7 // indirect
Expand All @@ -96,7 +96,7 @@ require (
github.com/pion/srtp/v2 v2.0.16 // indirect
github.com/pion/stun v0.6.1 // indirect
github.com/pion/transport/v2 v2.2.1 // indirect
github.com/pion/turn/v2 v2.1.2 // indirect
github.com/pion/turn/v2 v2.1.3 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
Expand All @@ -111,8 +111,8 @@ require (
go.uber.org/goleak v1.1.12 // indirect
go.uber.org/multierr v1.6.0 // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b // indirect
golang.org/x/net v0.12.0 // indirect
golang.org/x/exp v0.0.0-20230810033253-352e893a4cad // indirect
golang.org/x/net v0.13.0 // indirect
golang.org/x/oauth2 v0.9.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.10.0 // indirect
Expand Down
25 changes: 14 additions & 11 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,18 +171,18 @@ github.com/livekit/go-glib v0.0.0-20230223001336-834490045522 h1:AlU57PAPgzde7q9
github.com/livekit/go-glib v0.0.0-20230223001336-834490045522/go.mod h1:ltV0gO6xNFzZhsIRbFXv8RTq9NGoNT2dmAER4YmZfaM=
github.com/livekit/go-gst v0.2.34-0.20230623013326-3ca8cb8f62e5 h1:psIQLAp+yJETA+vZTVphV072nvCgcwC55xcufo1I5v0=
github.com/livekit/go-gst v0.2.34-0.20230623013326-3ca8cb8f62e5/go.mod h1:0hI+orMYVT61TEh429LvmoV9UmyqjeTqdJ3DW2TX114=
github.com/livekit/livekit-server v1.4.4-0.20230612120056-afa773374840 h1:bi2ZWnpdrSu18Tx344mTIoRO0qSVqgvCC++nAADxto4=
github.com/livekit/livekit-server v1.4.4-0.20230612120056-afa773374840/go.mod h1:V8OlzLcbkG/8rQLLphsT379avf/CvXHRW8YlslL9q2M=
github.com/livekit/livekit-server v1.4.5-0.20230814182001-77c8e824735b h1:o+fyF5iijKHAMbzXFW6g4pRH/bR/cELRbTZHwRheF24=
github.com/livekit/livekit-server v1.4.5-0.20230814182001-77c8e824735b/go.mod h1:Dbul6eFPxLw/ItxGjovO1a2Gzrs2EHEuXgjADz6squI=
github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58=
github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ=
github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a h1:JWpPHcMFuw0fP4swE89CfMgeUXiSN5IKvCJL/5HLI3A=
github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a/go.mod h1:xirUXW8xnLGmfCwUeAv/nj1VGo1OO1BmgxrYP7jK/14=
github.com/livekit/mediatransportutil v0.0.0-20230814030822-8d5de0008b08 h1:e0qwVjrtzmADgNZpdgSJgyhlF6BgrHkpdnkONL8pLrw=
github.com/livekit/mediatransportutil v0.0.0-20230814030822-8d5de0008b08/go.mod h1:xirUXW8xnLGmfCwUeAv/nj1VGo1OO1BmgxrYP7jK/14=
github.com/livekit/protocol v1.6.0 h1:19S+vFZqnivKIOpyR3DEK/mSaykQ3UEf7H2G/mBOE54=
github.com/livekit/protocol v1.6.0/go.mod h1:SUS9foM1xBzw/AFrgTJuFX/oSuwlnIbHmpdiPdCvwEM=
github.com/livekit/psrpc v0.3.2 h1:eAaJhASme33gtoBhCRLH9jsnWcdm1tHWf0WzaDk56ew=
github.com/livekit/psrpc v0.3.2/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U=
github.com/livekit/server-sdk-go v1.0.16-0.20230811173715-bab67d4a7670 h1:s22ZySAWzFymYn4B577XfyfDGULFJbyzu2nzohOA1GM=
github.com/livekit/server-sdk-go v1.0.16-0.20230811173715-bab67d4a7670/go.mod h1:4OIkcDpQJuOQJEThr+Z8+WQfkcs12BXC+9cuhigLnDM=
github.com/livekit/server-sdk-go v1.0.16-0.20230815025737-c12cd2eb8fe8 h1:Vm64l3U2rWAB2IdG/0/CXIZTgVPmldsQrDAOcq+zQQs=
github.com/livekit/server-sdk-go v1.0.16-0.20230815025737-c12cd2eb8fe8/go.mod h1:4OIkcDpQJuOQJEThr+Z8+WQfkcs12BXC+9cuhigLnDM=
github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs=
github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ=
github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
Expand Down Expand Up @@ -219,8 +219,9 @@ github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew
github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0=
github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8=
github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
github.com/pion/ice/v2 v2.3.9 h1:7yZpHf3PhPxJGT4JkMj1Y8Rl5cQ6fB709iz99aeMd/U=
github.com/pion/ice/v2 v2.3.9/go.mod h1:lT3kv5uUIlHfXHU/ZRD7uKD/ufM202+eTa3C/umgGf4=
github.com/pion/ice/v2 v2.3.10 h1:T3bUJKqh7pGEdMyTngUcTeQd6io9X8JjgsVWZDannnY=
github.com/pion/ice/v2 v2.3.10/go.mod h1:hHGCibDfmXGqukayQw979xEctASp2Pe5Oe0iDU8pRus=
github.com/pion/interceptor v0.1.17 h1:prJtgwFh/gB8zMqGZoOgJPHivOwVAp61i2aG61Du/1w=
github.com/pion/interceptor v0.1.17/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI=
github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
Expand Down Expand Up @@ -251,8 +252,9 @@ github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlD
github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ=
github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c=
github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g=
github.com/pion/turn/v2 v2.1.2 h1:wj0cAoGKltaZ790XEGW9HwoUewqjliwmhtxCuB2ApyM=
github.com/pion/turn/v2 v2.1.2/go.mod h1:1kjnPkBcex3dhCU2Am+AAmxDcGhLX3WnMfmkNpvSTQU=
github.com/pion/turn/v2 v2.1.3 h1:pYxTVWG2gpC97opdRc5IGsQ1lJ9O/IlNhkzj7MMrGAA=
github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
github.com/pion/webrtc/v3 v3.2.14 h1:GlqnBnnLlcYYA/LOwqLLU1plZYwx0Y/e/57bZ2tzQcU=
github.com/pion/webrtc/v3 v3.2.14/go.mod h1:r1mtixc2MH847mmQTPwlEvGge7D18C2T5qp8jI9Lm44=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
Expand Down Expand Up @@ -329,8 +331,8 @@ golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI=
golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggkk1bJDsINcuWmJN1iJU=
golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
Expand Down Expand Up @@ -366,8 +368,9 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY=
golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs=
Expand Down
13 changes: 7 additions & 6 deletions pkg/config/pipeline.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,12 @@ type SDKSourceParams struct {
}

type TrackSource struct {
TrackID string
Kind lksdk.TrackKind
AppSrc *app.Source
Codec webrtc.RTPCodecParameters
TrackID string
Kind lksdk.TrackKind
AppSrc *app.Source
MimeType types.MimeType
PayloadType webrtc.PayloadType
ClockRate uint32
}

type AudioConfig struct {
Expand Down Expand Up @@ -390,8 +392,7 @@ func (p *PipelineConfig) Update(request *rpc.StartEgressRequest) error {
}
}

if p.TrackID == "" {
// Track egress output format decision happens after join
if p.RequestType != types.RequestTypeTrack {
err := p.validateAndUpdateOutputParams()
if err != nil {
return err
Expand Down
1 change: 0 additions & 1 deletion pkg/errors/errors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ var (
ErrNoCompatibleCodec = psrpc.NewErrorf(psrpc.InvalidArgument, "no supported codec is compatible with all outputs")
ErrNoCompatibleFileOutputType = psrpc.NewErrorf(psrpc.InvalidArgument, "no supported file output type is compatible with the selected codecs")
ErrResourceExhausted = psrpc.NewErrorf(psrpc.ResourceExhausted, "not enough CPU")
ErrVideoWebsocket = psrpc.NewErrorf(psrpc.InvalidArgument, "cannot send video over websocket")
ErrInvalidTrack = psrpc.NewErrorf(psrpc.Internal, "unexpected track type")
)

Expand Down
24 changes: 11 additions & 13 deletions pkg/pipeline/input/audio.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package input

import (
"fmt"
"strings"

"github.com/tinyzimmer/go-gst/gst"

Expand Down Expand Up @@ -72,14 +71,12 @@ func (a *audioInput) buildAppSource(track *config.TrackSource) error {
}
a.src = []*gst.Element{track.AppSrc.Element}

switch {
case strings.EqualFold(track.Codec.MimeType, string(types.MimeTypeOpus)):
if err := track.AppSrc.Element.SetProperty("caps", gst.NewCapsFromString(
fmt.Sprintf(
"application/x-rtp,media=audio,payload=%d,encoding-name=OPUS,clock-rate=%d",
track.Codec.PayloadType, track.Codec.ClockRate,
),
)); err != nil {
switch track.MimeType {
case types.MimeTypeOpus:
if err := track.AppSrc.Element.SetProperty("caps", gst.NewCapsFromString(fmt.Sprintf(
"application/x-rtp,media=audio,payload=%d,encoding-name=OPUS,clock-rate=%d",
track.PayloadType, track.ClockRate,
))); err != nil {
return errors.ErrGstPipelineError(err)
}

Expand All @@ -96,7 +93,7 @@ func (a *audioInput) buildAppSource(track *config.TrackSource) error {
a.src = append(a.src, rtpOpusDepay, opusDec)

default:
return errors.ErrNotSupported(track.Codec.MimeType)
return errors.ErrNotSupported(string(track.MimeType))
}

return nil
Expand Down Expand Up @@ -209,9 +206,10 @@ func newAudioCapsFilter(p *config.PipelineConfig) (*gst.Element, error) {
"audio/x-raw,format=S16LE,layout=interleaved,rate=48000,channels=2",
)
case types.MimeTypeAAC:
caps = gst.NewCapsFromString(
fmt.Sprintf("audio/x-raw,format=S16LE,layout=interleaved,rate=%d,channels=2", p.AudioFrequency),
)
caps = gst.NewCapsFromString(fmt.Sprintf(
"audio/x-raw,format=S16LE,layout=interleaved,rate=%d,channels=2",
p.AudioFrequency,
))
default:
return nil, errors.ErrNotSupported(string(p.AudioOutCodec))
}
Expand Down
Loading

0 comments on commit cf060cc

Please sign in to comment.