From aa31f5727dde751dc49c631e81eef706844dedc5 Mon Sep 17 00:00:00 2001 From: aler9 <46489434+aler9@users.noreply.github.com> Date: Sat, 27 Apr 2024 20:37:37 +0200 Subject: [PATCH] Fix VP9 decoding on iOS The current implementation of the VP9 payloader produces payloads that are not compatible with iOS. This is because the payloader provides only the muxing strategy called "flexible mode". According to the VP9 RFC draft, there are two ways to wrap VP9 frames into RTP packets: the "flexible mode" and the "non-flexible mode", with the latter being the preferred one for live-streaming applications. In particular, all browsers encodes VP9 RTP packets in the "non-flexible mode", while iOS supports decoding RTP packets in this mode only, and this is probably a problem shared by other implementations. This patch improves the VP9 payloader by adding support for the "non-flexible mode". The "flexible mode" is retained and a flag is provided to perform the switch between the two modes. --- codecs/vp9/bits.go | 65 ++++++ codecs/vp9/header.go | 220 ++++++++++++++++++ codecs/vp9/header_test.go | 93 ++++++++ .../fuzz/FuzzHeaderUnmarshal/02ff90541f5f1194 | 2 + .../fuzz/FuzzHeaderUnmarshal/26f4501756141118 | 2 + .../fuzz/FuzzHeaderUnmarshal/32c8b8b4246d92a8 | 2 + .../fuzz/FuzzHeaderUnmarshal/49ada3d55320a56f | 2 + .../fuzz/FuzzHeaderUnmarshal/582528ddfad69eb5 | 2 + .../fuzz/FuzzHeaderUnmarshal/5c051c9165823a32 | 2 + .../fuzz/FuzzHeaderUnmarshal/8fb58564ec995b50 | 2 + .../fuzz/FuzzHeaderUnmarshal/9623e027b3ad6e3f | 2 + .../fuzz/FuzzHeaderUnmarshal/caf81e9797b19c76 | 2 + .../fuzz/FuzzHeaderUnmarshal/e2b7ed47d80236aa | 2 + .../fuzz/FuzzHeaderUnmarshal/efafd5efc9220ade | 2 + codecs/vp9_packet.go | 153 +++++++++--- codecs/vp9_packet_test.go | 135 ++++++++--- 16 files changed, 628 insertions(+), 60 deletions(-) create mode 100644 codecs/vp9/bits.go create mode 100644 codecs/vp9/header.go create mode 100644 codecs/vp9/header_test.go create mode 100644 codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/02ff90541f5f1194 create mode 100644 codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/26f4501756141118 create mode 100644 codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/32c8b8b4246d92a8 create mode 100644 codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/49ada3d55320a56f create mode 100644 codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/582528ddfad69eb5 create mode 100644 codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/5c051c9165823a32 create mode 100644 codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/8fb58564ec995b50 create mode 100644 codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/9623e027b3ad6e3f create mode 100644 codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/caf81e9797b19c76 create mode 100644 codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/e2b7ed47d80236aa create mode 100644 codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/efafd5efc9220ade diff --git a/codecs/vp9/bits.go b/codecs/vp9/bits.go new file mode 100644 index 0000000..a6a3c1f --- /dev/null +++ b/codecs/vp9/bits.go @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: 2023 The Pion community +// SPDX-License-Identifier: MIT + +package vp9 + +import "errors" + +var errNotEnoughBits = errors.New("not enough bits") + +func hasSpace(buf []byte, pos int, n int) error { + if n > ((len(buf) * 8) - pos) { + return errNotEnoughBits + } + return nil +} + +func readFlag(buf []byte, pos *int) (bool, error) { + err := hasSpace(buf, *pos, 1) + if err != nil { + return false, err + } + + return readFlagUnsafe(buf, pos), nil +} + +func readFlagUnsafe(buf []byte, pos *int) bool { + b := (buf[*pos>>0x03] >> (7 - (*pos & 0x07))) & 0x01 + *pos++ + return b == 1 +} + +func readBits(buf []byte, pos *int, n int) (uint64, error) { + err := hasSpace(buf, *pos, n) + if err != nil { + return 0, err + } + + return readBitsUnsafe(buf, pos, n), nil +} + +func readBitsUnsafe(buf []byte, pos *int, n int) uint64 { + res := 8 - (*pos & 0x07) + if n < res { + v := uint64((buf[*pos>>0x03] >> (res - n)) & (1<>0x03] & (1<= 8 { + v = (v << 8) | uint64(buf[*pos>>0x03]) + *pos += 8 + n -= 8 + } + + if n > 0 { + v = (v << n) | uint64(buf[*pos>>0x03]>>(8-n)) + *pos += n + } + + return v +} diff --git a/codecs/vp9/header.go b/codecs/vp9/header.go new file mode 100644 index 0000000..be402b0 --- /dev/null +++ b/codecs/vp9/header.go @@ -0,0 +1,220 @@ +// SPDX-FileCopyrightText: 2023 The Pion community +// SPDX-License-Identifier: MIT + +// Package vp9 contains a VP9 header parser. +package vp9 + +import ( + "errors" +) + +var ( + errInvalidFrameMarker = errors.New("invalid frame marker") + errWrongFrameSyncByte0 = errors.New("wrong frame_sync_byte_0") + errWrongFrameSyncByte1 = errors.New("wrong frame_sync_byte_1") + errWrongFrameSyncByte2 = errors.New("wrong frame_sync_byte_2") +) + +// HeaderColorConfig is the color_config member of an header. +type HeaderColorConfig struct { + TenOrTwelveBit bool + BitDepth uint8 + ColorSpace uint8 + ColorRange bool + SubsamplingX bool + SubsamplingY bool +} + +func (c *HeaderColorConfig) unmarshal(profile uint8, buf []byte, pos *int) error { + if profile >= 2 { + var err error + c.TenOrTwelveBit, err = readFlag(buf, pos) + if err != nil { + return err + } + + if c.TenOrTwelveBit { + c.BitDepth = 12 + } else { + c.BitDepth = 10 + } + } else { + c.BitDepth = 8 + } + + tmp, err := readBits(buf, pos, 3) + if err != nil { + return err + } + c.ColorSpace = uint8(tmp) + + if c.ColorSpace != 7 { + var err error + c.ColorRange, err = readFlag(buf, pos) + if err != nil { + return err + } + + if profile == 1 || profile == 3 { + err := hasSpace(buf, *pos, 3) + if err != nil { + return err + } + + c.SubsamplingX = readFlagUnsafe(buf, pos) + c.SubsamplingY = readFlagUnsafe(buf, pos) + *pos++ + } else { + c.SubsamplingX = true + c.SubsamplingY = true + } + } else { + c.ColorRange = true + + if profile == 1 || profile == 3 { + c.SubsamplingX = false + c.SubsamplingY = false + + err := hasSpace(buf, *pos, 1) + if err != nil { + return err + } + *pos++ + } + } + + return nil +} + +// HeaderFrameSize is the frame_size member of an header. +type HeaderFrameSize struct { + FrameWidthMinus1 uint16 + FrameHeightMinus1 uint16 +} + +func (s *HeaderFrameSize) unmarshal(buf []byte, pos *int) error { + err := hasSpace(buf, *pos, 32) + if err != nil { + return err + } + + s.FrameWidthMinus1 = uint16(readBitsUnsafe(buf, pos, 16)) + s.FrameHeightMinus1 = uint16(readBitsUnsafe(buf, pos, 16)) + return nil +} + +// Header is a VP9 Frame header. +// Specification: +// https://storage.googleapis.com/downloads.webmproject.org/docs/vp9/vp9-bitstream-specification-v0.6-20160331-draft.pdf +type Header struct { + Profile uint8 + ShowExistingFrame bool + FrameToShowMapIdx uint8 + NonKeyFrame bool + ShowFrame bool + ErrorResilientMode bool + ColorConfig *HeaderColorConfig + FrameSize *HeaderFrameSize +} + +// Unmarshal decodes a Header. +func (h *Header) Unmarshal(buf []byte) error { + pos := 0 + + err := hasSpace(buf, pos, 4) + if err != nil { + return err + } + + frameMarker := readBitsUnsafe(buf, &pos, 2) + if frameMarker != 2 { + return errInvalidFrameMarker + } + + profileLowBit := uint8(readBitsUnsafe(buf, &pos, 1)) + profileHighBit := uint8(readBitsUnsafe(buf, &pos, 1)) + h.Profile = profileHighBit<<1 + profileLowBit + + if h.Profile == 3 { + err := hasSpace(buf, pos, 1) + if err != nil { + return err + } + pos++ + } + + h.ShowExistingFrame, err = readFlag(buf, &pos) + if err != nil { + return err + } + + if h.ShowExistingFrame { + tmp, err := readBits(buf, &pos, 3) + if err != nil { + return err + } + h.FrameToShowMapIdx = uint8(tmp) + return nil + } + + err = hasSpace(buf, pos, 3) + if err != nil { + return err + } + + h.NonKeyFrame = readFlagUnsafe(buf, &pos) + h.ShowFrame = readFlagUnsafe(buf, &pos) + h.ErrorResilientMode = readFlagUnsafe(buf, &pos) + + if !h.NonKeyFrame { + err := hasSpace(buf, pos, 24) + if err != nil { + return err + } + + frameSyncByte0 := uint8(readBitsUnsafe(buf, &pos, 8)) + if frameSyncByte0 != 0x49 { + return errWrongFrameSyncByte0 + } + + frameSyncByte1 := uint8(readBitsUnsafe(buf, &pos, 8)) + if frameSyncByte1 != 0x83 { + return errWrongFrameSyncByte1 + } + + frameSyncByte2 := uint8(readBitsUnsafe(buf, &pos, 8)) + if frameSyncByte2 != 0x42 { + return errWrongFrameSyncByte2 + } + + h.ColorConfig = &HeaderColorConfig{} + err = h.ColorConfig.unmarshal(h.Profile, buf, &pos) + if err != nil { + return err + } + + h.FrameSize = &HeaderFrameSize{} + err = h.FrameSize.unmarshal(buf, &pos) + if err != nil { + return err + } + } + + return nil +} + +// Width returns the video width. +func (h Header) Width() uint16 { + if h.FrameSize == nil { + return 0 + } + return h.FrameSize.FrameWidthMinus1 + 1 +} + +// Height returns the video height. +func (h Header) Height() uint16 { + if h.FrameSize == nil { + return 0 + } + return h.FrameSize.FrameHeightMinus1 + 1 +} diff --git a/codecs/vp9/header_test.go b/codecs/vp9/header_test.go new file mode 100644 index 0000000..a33ec6e --- /dev/null +++ b/codecs/vp9/header_test.go @@ -0,0 +1,93 @@ +package vp9 + +import ( + "reflect" + "testing" +) + +var cases = []struct { + name string + byts []byte + sh Header + width uint16 + height uint16 +}{ + { + "chrome webrtc", + []byte{ + 0x82, 0x49, 0x83, 0x42, 0x00, 0x77, 0xf0, 0x32, + 0x34, 0x30, 0x38, 0x24, 0x1c, 0x19, 0x40, 0x18, + 0x03, 0x40, 0x5f, 0xb4, + }, + Header{ + ShowFrame: true, + ColorConfig: &HeaderColorConfig{ + BitDepth: 8, + SubsamplingX: true, + SubsamplingY: true, + }, + FrameSize: &HeaderFrameSize{ + FrameWidthMinus1: 1919, + FrameHeightMinus1: 803, + }, + }, + 1920, + 804, + }, + { + "vp9 sample", + []byte{ + 0x82, 0x49, 0x83, 0x42, 0x40, 0xef, 0xf0, 0x86, + 0xf4, 0x04, 0x21, 0xa0, 0xe0, 0x00, 0x30, 0x70, + 0x00, 0x00, 0x00, 0x01, + }, + Header{ + ShowFrame: true, + ColorConfig: &HeaderColorConfig{ + BitDepth: 8, + ColorSpace: 2, + SubsamplingX: true, + SubsamplingY: true, + }, + FrameSize: &HeaderFrameSize{ + FrameWidthMinus1: 3839, + FrameHeightMinus1: 2159, + }, + }, + 3840, + 2160, + }, +} + +func TestHeaderUnmarshal(t *testing.T) { + for _, ca := range cases { + t.Run(ca.name, func(t *testing.T) { + var sh Header + err := sh.Unmarshal(ca.byts) + if err != nil { + t.Fatal("unexpected error") + } + + if !reflect.DeepEqual(ca.sh, sh) { + t.Fatalf("expected %#+v, got %#+v", ca.sh, sh) + } + if ca.width != sh.Width() { + t.Fatalf("unexpected width") + } + if ca.height != sh.Height() { + t.Fatalf("unexpected height") + } + }) + } +} + +func FuzzHeaderUnmarshal(f *testing.F) { + for _, ca := range cases { + f.Add(ca.byts) + } + + f.Fuzz(func(_ *testing.T, b []byte) { + var sh Header + sh.Unmarshal(b) //nolint:errcheck + }) +} diff --git a/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/02ff90541f5f1194 b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/02ff90541f5f1194 new file mode 100644 index 0000000..c6bfed7 --- /dev/null +++ b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/02ff90541f5f1194 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\xa3000") diff --git a/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/26f4501756141118 b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/26f4501756141118 new file mode 100644 index 0000000..7c44c30 --- /dev/null +++ b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/26f4501756141118 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\xa3I00") diff --git a/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/32c8b8b4246d92a8 b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/32c8b8b4246d92a8 new file mode 100644 index 0000000..16cb300 --- /dev/null +++ b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/32c8b8b4246d92a8 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\x91") diff --git a/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/49ada3d55320a56f b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/49ada3d55320a56f new file mode 100644 index 0000000..23278af --- /dev/null +++ b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/49ada3d55320a56f @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\xa3I\x83B") diff --git a/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/582528ddfad69eb5 b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/582528ddfad69eb5 new file mode 100644 index 0000000..a96f559 --- /dev/null +++ b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/582528ddfad69eb5 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0") diff --git a/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/5c051c9165823a32 b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/5c051c9165823a32 new file mode 100644 index 0000000..50497b1 --- /dev/null +++ b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/5c051c9165823a32 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\xa3I\x830") diff --git a/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/8fb58564ec995b50 b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/8fb58564ec995b50 new file mode 100644 index 0000000..39022c6 --- /dev/null +++ b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/8fb58564ec995b50 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\xb0") diff --git a/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/9623e027b3ad6e3f b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/9623e027b3ad6e3f new file mode 100644 index 0000000..ec5db24 --- /dev/null +++ b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/9623e027b3ad6e3f @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\xbd") diff --git a/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/caf81e9797b19c76 b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/caf81e9797b19c76 new file mode 100644 index 0000000..67322c7 --- /dev/null +++ b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/caf81e9797b19c76 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("") diff --git a/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/e2b7ed47d80236aa b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/e2b7ed47d80236aa new file mode 100644 index 0000000..89d64d4 --- /dev/null +++ b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/e2b7ed47d80236aa @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\xa3I\x83B0") diff --git a/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/efafd5efc9220ade b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/efafd5efc9220ade new file mode 100644 index 0000000..cae9cf7 --- /dev/null +++ b/codecs/vp9/testdata/fuzz/FuzzHeaderUnmarshal/efafd5efc9220ade @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\x92I\x83B") diff --git a/codecs/vp9_packet.go b/codecs/vp9_packet.go index 6a6e0b0..a27964b 100644 --- a/codecs/vp9_packet.go +++ b/codecs/vp9_packet.go @@ -5,6 +5,7 @@ package codecs import ( "github.com/pion/randutil" + "github.com/pion/rtp/codecs/vp9" ) // Use global random generator to properly seed by crypto grade random. @@ -12,24 +13,50 @@ var globalMathRandomGenerator = randutil.NewMathRandomGenerator() // nolint:goch // VP9Payloader payloads VP9 packets type VP9Payloader struct { - pictureID uint16 - initialized bool + // whether to use flexible mode or non-flexible mode. + FlexibleMode bool // InitialPictureIDFn is a function that returns random initial picture ID. InitialPictureIDFn func() uint16 + + pictureID uint16 + initialized bool } const ( - vp9HeaderSize = 3 // Flexible mode 15 bit picture ID maxSpatialLayers = 5 maxVP9RefPics = 3 ) // Payload fragments an VP9 packet across one or more byte arrays func (p *VP9Payloader) Payload(mtu uint16, payload []byte) [][]byte { + if !p.initialized { + if p.InitialPictureIDFn == nil { + p.InitialPictureIDFn = func() uint16 { + return uint16(globalMathRandomGenerator.Intn(0x7FFF)) + } + } + p.pictureID = p.InitialPictureIDFn() & 0x7FFF + p.initialized = true + } + + var payloads [][]byte + if p.FlexibleMode { + payloads = p.payloadFlexible(mtu, payload) + } else { + payloads = p.payloadNonFlexible(mtu, payload) + } + + p.pictureID++ + if p.pictureID >= 0x8000 { + p.pictureID = 0 + } + + return payloads +} + +func (p *VP9Payloader) payloadFlexible(mtu uint16, payload []byte) [][]byte { /* - * https://www.ietf.org/id/draft-ietf-payload-vp9-13.txt - * * Flexible mode (F=1) * 0 1 2 3 4 5 6 7 * +-+-+-+-+-+-+-+-+ @@ -46,7 +73,45 @@ func (p *VP9Payloader) Payload(mtu uint16, payload []byte) [][]byte { * V: | SS | * | .. | * +-+-+-+-+-+-+-+-+ - * + */ + + headerSize := 3 + maxFragmentSize := int(mtu) - headerSize + payloadDataRemaining := len(payload) + payloadDataIndex := 0 + var payloads [][]byte + + if min(maxFragmentSize, payloadDataRemaining) <= 0 { + return [][]byte{} + } + + for payloadDataRemaining > 0 { + currentFragmentSize := min(maxFragmentSize, payloadDataRemaining) + out := make([]byte, headerSize+currentFragmentSize) + + out[0] = 0x90 // F=1, I=1 + if payloadDataIndex == 0 { + out[0] |= 0x08 // B=1 + } + if payloadDataRemaining == currentFragmentSize { + out[0] |= 0x04 // E=1 + } + + out[1] = byte(p.pictureID>>8) | 0x80 + out[2] = byte(p.pictureID) + + copy(out[headerSize:], payload[payloadDataIndex:payloadDataIndex+currentFragmentSize]) + payloads = append(payloads, out) + + payloadDataRemaining -= currentFragmentSize + payloadDataIndex += currentFragmentSize + } + + return payloads +} + +func (p *VP9Payloader) payloadNonFlexible(mtu uint16, payload []byte) [][]byte { + /* * Non-flexible mode (F=0) * 0 1 2 3 4 5 6 7 * +-+-+-+-+-+-+-+-+ @@ -65,51 +130,81 @@ func (p *VP9Payloader) Payload(mtu uint16, payload []byte) [][]byte { * +-+-+-+-+-+-+-+-+ */ - if !p.initialized { - if p.InitialPictureIDFn == nil { - p.InitialPictureIDFn = func() uint16 { - return uint16(globalMathRandomGenerator.Intn(0x7FFF)) - } - } - p.pictureID = p.InitialPictureIDFn() & 0x7FFF - p.initialized = true - } - if payload == nil { + var h vp9.Header + err := h.Unmarshal(payload) + if err != nil { return [][]byte{} } - maxFragmentSize := int(mtu) - vp9HeaderSize payloadDataRemaining := len(payload) payloadDataIndex := 0 - - if min(maxFragmentSize, payloadDataRemaining) <= 0 { - return [][]byte{} - } - var payloads [][]byte + for payloadDataRemaining > 0 { + var headerSize int + if !h.NonKeyFrame && payloadDataIndex == 0 { + headerSize = 3 + 8 + } else { + headerSize = 3 + } + + maxFragmentSize := int(mtu) - headerSize currentFragmentSize := min(maxFragmentSize, payloadDataRemaining) - out := make([]byte, vp9HeaderSize+currentFragmentSize) + if currentFragmentSize <= 0 { + return [][]byte{} + } + + out := make([]byte, headerSize+currentFragmentSize) + + out[0] = 0x80 | 0x01 // I=1, Z=1 - out[0] = 0x90 // F=1 I=1 + if h.NonKeyFrame { + out[0] |= 0x40 // P=1 + } if payloadDataIndex == 0 { out[0] |= 0x08 // B=1 } if payloadDataRemaining == currentFragmentSize { out[0] |= 0x04 // E=1 } + out[1] = byte(p.pictureID>>8) | 0x80 out[2] = byte(p.pictureID) - copy(out[vp9HeaderSize:], payload[payloadDataIndex:payloadDataIndex+currentFragmentSize]) + off := 3 + + if !h.NonKeyFrame && payloadDataIndex == 0 { + out[0] |= 0x02 // V=1 + out[off] = 0x10 | 0x08 // N_S=0, Y=1, G=1 + off++ + + width := h.Width() + out[off] = byte(width >> 8) + off++ + out[off] = byte(width & 0xFF) + off++ + + height := h.Height() + out[off] = byte(height >> 8) + off++ + out[off] = byte(height & 0xFF) + off++ + + out[off] = 0x01 // N_G=1 + off++ + + out[off] = 1<<4 | 1<<2 // TID=0, U=1, R=1 + off++ + + out[off] = 0x01 // P_DIFF=1 + off++ + } + + copy(out[headerSize:], payload[payloadDataIndex:payloadDataIndex+currentFragmentSize]) payloads = append(payloads, out) payloadDataRemaining -= currentFragmentSize payloadDataIndex += currentFragmentSize } - p.pictureID++ - if p.pictureID >= 0x8000 { - p.pictureID = 0 - } return payloads } diff --git a/codecs/vp9_packet_test.go b/codecs/vp9_packet_test.go index 97e176b..9591f22 100644 --- a/codecs/vp9_packet_test.go +++ b/codecs/vp9_packet_test.go @@ -5,7 +5,6 @@ package codecs import ( "errors" - "fmt" "math/rand" "reflect" "testing" @@ -223,62 +222,134 @@ func TestVP9Payloader_Payload(t *testing.T) { } cases := map[string]struct { - b [][]byte - mtu uint16 - res [][]byte + b [][]byte + flexible bool + mtu uint16 + res [][]byte }{ - "NilPayload": { - b: [][]byte{nil}, - mtu: 100, - res: [][]byte{}, + "flexible NilPayload": { + b: [][]byte{nil}, + flexible: true, + mtu: 100, + res: [][]byte{}, }, - "SmallMTU": { - b: [][]byte{{0x00, 0x00}}, - mtu: 1, - res: [][]byte{}, + "flexible SmallMTU": { + b: [][]byte{{0x00, 0x00}}, + flexible: true, + mtu: 1, + res: [][]byte{}, }, - "OnePacket": { - b: [][]byte{{0x01, 0x02}}, - mtu: 10, + "flexible OnePacket": { + b: [][]byte{{0x01, 0x02}}, + flexible: true, + mtu: 10, res: [][]byte{ {0x9C, rands[0][0], rands[0][1], 0x01, 0x02}, }, }, - "TwoPackets": { - b: [][]byte{{0x01, 0x02}}, - mtu: 4, + "flexible TwoPackets": { + b: [][]byte{{0x01, 0x02}}, + flexible: true, + mtu: 4, res: [][]byte{ {0x98, rands[0][0], rands[0][1], 0x01}, {0x94, rands[0][0], rands[0][1], 0x02}, }, }, - "ThreePackets": { - b: [][]byte{{0x01, 0x02, 0x03}}, - mtu: 4, + "flexible ThreePackets": { + b: [][]byte{{0x01, 0x02, 0x03}}, + flexible: true, + mtu: 4, res: [][]byte{ {0x98, rands[0][0], rands[0][1], 0x01}, {0x90, rands[0][0], rands[0][1], 0x02}, {0x94, rands[0][0], rands[0][1], 0x03}, }, }, - "TwoFramesFourPackets": { - b: [][]byte{{0x01, 0x02, 0x03}, {0x04}}, - mtu: 5, + "flexible TwoFramesFourPackets": { + b: [][]byte{{0x01, 0x02, 0x03}, {0x04}}, + flexible: true, + mtu: 5, res: [][]byte{ {0x98, rands[0][0], rands[0][1], 0x01, 0x02}, {0x94, rands[0][0], rands[0][1], 0x03}, {0x9C, rands[1][0], rands[1][1], 0x04}, }, }, + "non-flexible NilPayload": { + b: [][]byte{nil}, + mtu: 100, + res: [][]byte{}, + }, + "non-flexible SmallMTU": { + b: [][]byte{{0x82, 0x49, 0x83, 0x42, 0x0, 0x77, 0xf0, 0x32, 0x34}}, + mtu: 1, + res: [][]byte{}, + }, + "non-flexible OnePacket key frame": { + b: [][]byte{{0x82, 0x49, 0x83, 0x42, 0x0, 0x77, 0xf0, 0x32, 0x34}}, + mtu: 20, + res: [][]byte{{ + 0x8f, 0xa1, 0xf4, 0x18, 0x07, 0x80, 0x03, 0x24, + 0x01, 0x14, 0x01, 0x82, 0x49, 0x83, 0x42, 0x00, + 0x77, 0xf0, 0x32, 0x34, + }}, + }, + "non-flexible TwoPackets key frame": { + b: [][]byte{{0x82, 0x49, 0x83, 0x42, 0x0, 0x77, 0xf0, 0x32, 0x34}}, + mtu: 12, + res: [][]byte{ + { + 0x8b, 0xa1, 0xf4, 0x18, 0x07, 0x80, 0x03, 0x24, + 0x01, 0x14, 0x01, 0x82, + }, + { + 0x85, 0xa1, 0xf4, 0x49, 0x83, 0x42, 0x00, 0x77, + 0xf0, 0x32, 0x34, + }, + }, + }, + "non-flexible ThreePackets key frame": { + b: [][]byte{{ + 0x82, 0x49, 0x83, 0x42, 0x00, 0x77, 0xf0, 0x32, + 0x34, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, + }}, + mtu: 12, + res: [][]byte{ + { + 0x8b, 0xa1, 0xf4, 0x18, 0x07, 0x80, 0x03, 0x24, + 0x01, 0x14, 0x01, 0x82, + }, + { + 0x81, 0xa1, 0xf4, 0x49, 0x83, 0x42, 0x00, 0x77, + 0xf0, 0x32, 0x34, 0x01, + }, + { + 0x85, 0xa1, 0xf4, 0x02, 0x03, 0x04, 0x05, 0x06, + 0x07, 0x08, + }, + }, + }, + "non-flexible OnePacket non key frame": { + b: [][]byte{{0x86, 0x0, 0x40, 0x92, 0xe1, 0x31, 0x42, 0x8c, 0xc0, 0x40}}, + mtu: 20, + res: [][]byte{{ + 0xcd, 0xa1, 0xf4, 0x86, 0x00, 0x40, 0x92, 0xe1, + 0x31, 0x42, 0x8c, 0xc0, 0x40, + }}, + }, } + for name, c := range cases { - pck := VP9Payloader{ - InitialPictureIDFn: func() uint16 { - return uint16(rand.New(rand.NewSource(0)).Int31n(0x7FFF)) //nolint:gosec - }, - } - c := c - t.Run(fmt.Sprintf("%s_MTU%d", name, c.mtu), func(t *testing.T) { + t.Run(name, func(t *testing.T) { + pck := VP9Payloader{ + FlexibleMode: c.flexible, + InitialPictureIDFn: func() uint16 { + return uint16(rand.New(rand.NewSource(0)).Int31n(0x7FFF)) //nolint:gosec + }, + } + res := [][]byte{} for _, b := range c.b { res = append(res, pck.Payload(c.mtu, b)...) @@ -288,8 +359,10 @@ func TestVP9Payloader_Payload(t *testing.T) { } }) } + t.Run("PictureIDOverflow", func(t *testing.T) { pck := VP9Payloader{ + FlexibleMode: true, InitialPictureIDFn: func() uint16 { return uint16(rand.New(rand.NewSource(0)).Int31n(0x7FFF)) //nolint:gosec },