Skip to content

Commit

Permalink
[media] Support decode-to-texture mode
Browse files Browse the repository at this point in the history
Bring up decode-to-texture mode on Chrobalt.

b/375070492
  • Loading branch information
borongc committed Feb 6, 2025
1 parent 6d680b8 commit 8ef11b5
Show file tree
Hide file tree
Showing 5 changed files with 107 additions and 41 deletions.
4 changes: 1 addition & 3 deletions media/starboard/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,6 @@ source_set("starboard") {
# TODO(b/375069564): Revisit CValStats
"COBALT_MEDIA_ENABLE_CVAL=0",

# TODO(b/375070492): Revisit decode-to-texture
"COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER=0",

# TODO(b/375218518): Revisit FormatSupportQueryMetrics
"COBALT_MEDIA_ENABLE_FORMAT_SUPPORT_QUERY_METRICS=0",

Expand Down Expand Up @@ -77,6 +74,7 @@ source_set("starboard") {
sources += [
"bidirectional_fit_reuse_allocator.cc",
"bidirectional_fit_reuse_allocator.h",
"decode_target_provider.h",
"decoder_buffer_allocator.cc",
"decoder_buffer_allocator.h",
"decoder_buffer_memory_info.h",
Expand Down
93 changes: 93 additions & 0 deletions media/starboard/decode_target_provider.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
// Copyright 2018 The Cobalt Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef MEDIA_STARBOARD_DECODE_TARGET_PROVIDER_H_
#define MEDIA_STARBOARD_DECODE_TARGET_PROVIDER_H_

#include "base/functional/callback.h"
#include "base/memory/ref_counted.h"
#include "base/synchronization/lock.h"
#include "base/time/time.h"
#include "starboard/decode_target.h"

namespace media {

// The DecodeTargetProvider manages the backlog for video frames. It has the
// following functionalities:
// 1. It caches the video frames ready to be displayed.
// 2. It decides which frame to be displayed at the current time.
// 3. It removes frames that will no longer be displayed.
class DecodeTargetProvider
: public base::RefCountedThreadSafe<DecodeTargetProvider> {
public:
enum OutputMode {
kOutputModePunchOut,
kOutputModeDecodeToTexture,
kOutputModeInvalid,
};

DecodeTargetProvider() : output_mode_(kOutputModeInvalid) {}

DecodeTargetProvider(const DecodeTargetProvider&) = delete;
DecodeTargetProvider& operator=(const DecodeTargetProvider&) = delete;

using GetCurrentSbDecodeTargetFunction =
base::RepeatingCallback<SbDecodeTarget()>;

void SetOutputMode(OutputMode output_mode) {
base::AutoLock auto_lock(lock_);
output_mode_ = output_mode;
}

DecodeTargetProvider::OutputMode GetOutputMode() const {
base::AutoLock auto_lock(lock_);
return output_mode_;
}

// For Starboard platforms that have a decode-to-texture player, we enable
// this DecodeTargetProvider to act as a bridge for Cobalt code to query
// for the current SbDecodeTarget. In effect, we bypass all of
// DecodeTargetProvider's logic in this case, instead relying on the
// Starboard implementation to provide us with the current video frame when
// needed.
void SetGetCurrentSbDecodeTargetFunction(
GetCurrentSbDecodeTargetFunction function) {
base::AutoLock auto_lock(lock_);
get_current_sb_decode_target_function_ = function;
}

void ResetGetCurrentSbDecodeTargetFunction() {
base::AutoLock auto_lock(lock_);
get_current_sb_decode_target_function_.Reset();
}

SbDecodeTarget GetCurrentSbDecodeTarget() const {
base::AutoLock auto_lock(lock_);
if (get_current_sb_decode_target_function_.is_null()) {
return kSbDecodeTargetInvalid;
} else {
return get_current_sb_decode_target_function_.Run();
}
}

private:
mutable base::Lock lock_;

OutputMode output_mode_;
GetCurrentSbDecodeTargetFunction get_current_sb_decode_target_function_;
};

} // namespace media

#endif // MEDIA_STARBOARD_DECODE_TARGET_PROVIDER_H_
39 changes: 11 additions & 28 deletions media/starboard/sbplayer_bridge.cc
Original file line number Diff line number Diff line change
Expand Up @@ -203,9 +203,7 @@ SbPlayerBridge::SbPlayerBridge(
SbPlayerSetBoundsHelper* set_bounds_helper,
bool allow_resume_after_suspend,
SbPlayerOutputMode default_output_mode,
#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
DecodeTargetProvider* const decode_target_provider,
#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
const std::string& max_video_capabilities,
int max_video_input_size
#if COBALT_MEDIA_ENABLE_CVAL
Expand All @@ -225,9 +223,7 @@ SbPlayerBridge::SbPlayerBridge(
allow_resume_after_suspend_(allow_resume_after_suspend),
audio_config_(audio_config),
video_config_(video_config),
#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
decode_target_provider_(decode_target_provider),
#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
max_video_capabilities_(max_video_capabilities),
max_video_input_size_(max_video_input_size)
#if COBALT_MEDIA_ENABLE_CVAL
Expand All @@ -238,15 +234,11 @@ SbPlayerBridge::SbPlayerBridge(
is_url_based_(false)
#endif // SB_HAS(PLAYER_WITH_URL
{
#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
DCHECK(!get_decode_target_graphics_context_provider_func_.is_null());
#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
DCHECK(audio_config.IsValidConfig() || video_config.IsValidConfig());
DCHECK(host_);
DCHECK(set_bounds_helper_);
#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
DCHECK(decode_target_provider_);
#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER

audio_stream_info_.codec = kSbMediaAudioCodecNone;
video_stream_info_.codec = kSbMediaVideoCodecNone;
Expand All @@ -260,6 +252,11 @@ SbPlayerBridge::SbPlayerBridge(
}

output_mode_ = ComputeSbPlayerOutputMode(default_output_mode);
if (output_mode_ == kSbPlayerOutputModePunchOut) {
LOG(ERROR) << "Cobalt: kSbPlayerOutputModePunchOut";
} else if (output_mode_ == kSbPlayerOutputModeDecodeToTexture) {
LOG(ERROR) << "Cobalt: kSbPlayerOutputModeDecodeToTexture";
}

CreatePlayer();

Expand All @@ -279,11 +276,9 @@ SbPlayerBridge::~SbPlayerBridge() {
callback_helper_->ResetPlayer();
set_bounds_helper_->SetPlayerBridge(NULL);

#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
decode_target_provider_->SetOutputMode(
DecodeTargetProvider::kOutputModeInvalid);
decode_target_provider_->ResetGetCurrentSbDecodeTargetFunction();
#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER

if (SbPlayerIsValid(player_)) {
#if COBALT_MEDIA_ENABLE_CVAL
Expand Down Expand Up @@ -596,11 +591,9 @@ void SbPlayerBridge::Suspend() {

state_ = kSuspended;

#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
decode_target_provider_->SetOutputMode(
DecodeTargetProvider::kOutputModeInvalid);
decode_target_provider_->ResetGetCurrentSbDecodeTargetFunction();
#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER

#if COBALT_MEDIA_ENABLE_CVAL
cval_stats_->StartTimer(MediaTiming::SbPlayerDestroy, pipeline_identifier_);
Expand Down Expand Up @@ -648,8 +641,6 @@ void SbPlayerBridge::Resume(SbWindow window) {
}
}

#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER

namespace {

DecodeTargetProvider::OutputMode ToVideoFrameProviderOutputMode(
Expand All @@ -669,8 +660,6 @@ DecodeTargetProvider::OutputMode ToVideoFrameProviderOutputMode(

} // namespace

#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER

#if SB_HAS(PLAYER_WITH_URL)
// static
void SbPlayerBridge::EncryptedMediaInitDataEncounteredCB(
Expand Down Expand Up @@ -710,8 +699,9 @@ void SbPlayerBridge::CreateUrlPlayer(const std::string& url) {
if (output_mode_ == kSbPlayerOutputModeDecodeToTexture) {
// If the player is setup to decode to texture, then provide Cobalt with
// a method of querying that texture.
decode_target_provider_->SetGetCurrentSbDecodeTargetFunction(base::Bind(
&SbPlayerBridge::GetCurrentSbDecodeTarget, base::Unretained(this)));
decode_target_provider_->SetGetCurrentSbDecodeTargetFunction(
base::BindRepeating(&SbPlayerBridge::GetCurrentSbDecodeTarget,
base::Unretained(this)));
LOG(INFO) << "Playing in decode-to-texture mode.";
} else {
LOG(INFO) << "Playing in punch-out mode.";
Expand Down Expand Up @@ -790,11 +780,7 @@ void SbPlayerBridge::CreatePlayer() {
window_, &creation_param, &SbPlayerBridge::DeallocateSampleCB,
&SbPlayerBridge::DecoderStatusCB, &SbPlayerBridge::PlayerStatusCB,
&SbPlayerBridge::PlayerErrorCB, this,
#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
get_decode_target_graphics_context_provider_func_.Run());
#else // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
nullptr);
#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
#if COBALT_MEDIA_ENABLE_CVAL
cval_stats_->StopTimer(MediaTiming::SbPlayerCreate, pipeline_identifier_);
#endif // COBALT_MEDIA_ENABLE_CVAL
Expand All @@ -808,19 +794,16 @@ void SbPlayerBridge::CreatePlayer() {
if (output_mode_ == kSbPlayerOutputModeDecodeToTexture) {
// If the player is setup to decode to texture, then provide Cobalt with
// a method of querying that texture.
#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
decode_target_provider_->SetGetCurrentSbDecodeTargetFunction(base::Bind(
&SbPlayerBridge::GetCurrentSbDecodeTarget, base::Unretained(this)));
#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
decode_target_provider_->SetGetCurrentSbDecodeTargetFunction(
base::BindRepeating(&SbPlayerBridge::GetCurrentSbDecodeTarget,
base::Unretained(this)));
LOG(INFO) << "Playing in decode-to-texture mode.";
} else {
LOG(INFO) << "Playing in punch-out mode.";
}

#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
decode_target_provider_->SetOutputMode(
ToVideoFrameProviderOutputMode(output_mode_));
#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
set_bounds_helper_->SetPlayerBridge(this);

base::AutoLock auto_lock(lock_);
Expand Down
9 changes: 1 addition & 8 deletions media/starboard/sbplayer_bridge.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,6 @@
#include "cobalt/media/base/cval_stats.h"
#endif // COBALT_MEDIA_ENABLE_CVAL

#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
#include "cobalt/media/base/decode_target_provider.h"
#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER

#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME
#include "cobalt/media/base/decoder_buffer_cache.h"
#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME
Expand All @@ -43,6 +39,7 @@
#include "media/base/decoder_buffer.h"
#include "media/base/demuxer_stream.h"
#include "media/base/video_decoder_config.h"
#include "media/starboard/decode_target_provider.h"
#include "media/starboard/sbplayer_interface.h"
#include "media/starboard/sbplayer_set_bounds_helper.h"
#include "starboard/media.h"
Expand Down Expand Up @@ -101,9 +98,7 @@ class SbPlayerBridge {
SbPlayerSetBoundsHelper* set_bounds_helper,
bool allow_resume_after_suspend,
SbPlayerOutputMode default_output_mode,
#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
DecodeTargetProvider* const decode_target_provider,
#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
const std::string& max_video_capabilities,
int max_video_input_size
#if COBALT_MEDIA_ENABLE_CVAL
Expand Down Expand Up @@ -330,9 +325,7 @@ class SbPlayerBridge {
// Keep track of the output mode we are supposed to output to.
SbPlayerOutputMode output_mode_;

#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER
DecodeTargetProvider* const decode_target_provider_;
#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER

// Keep copies of the mime type strings instead of using the ones in the
// DemuxerStreams to ensure that the strings are always valid.
Expand Down
3 changes: 1 addition & 2 deletions media/starboard/starboard_renderer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -467,7 +467,6 @@ void StarboardRenderer::CreatePlayerBridge() {

player_bridge_.reset(new SbPlayerBridge(
&sbplayer_interface_, task_runner_,
// TODO(b/375070492): Implement decode-to-texture support
SbPlayerBridge::GetDecodeTargetGraphicsContextProviderFunc(),
audio_config, audio_mime_type, video_config, video_mime_type,
// TODO(b/326497953): Support suspend/resume.
Expand All @@ -477,7 +476,7 @@ void StarboardRenderer::CreatePlayerBridge() {
false,
// TODO(b/326825450): Revisit 360 videos.
// TODO(b/326827007): Support secondary videos.
kSbPlayerOutputModeInvalid,
kSbPlayerOutputModeInvalid, nullptr,
// TODO(b/326827007): Support secondary videos.
"",
// TODO(b/326654546): Revisit HTMLVideoElement.setMaxVideoInputSize.
Expand Down

0 comments on commit 8ef11b5

Please sign in to comment.