diff --git a/media/BUILD.gn b/media/BUILD.gn index ba14ef6f6b07..0f5ea42427f8 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -202,6 +202,10 @@ test("media_unittests") { "//media/webrtc:unit_tests", ] + if (is_cobalt) { + deps += ["//media/starboard:unit_tests"] + } + data = [ "test/data/", "formats/mp4/h264_annex_b_fuzz_corpus/", diff --git a/media/media_options.gni b/media/media_options.gni index cdc3e9375339..694f26f5e7fe 100644 --- a/media/media_options.gni +++ b/media/media_options.gni @@ -345,6 +345,10 @@ media_subcomponent_deps = [ "//media/video", ] +if (is_cobalt) { + media_subcomponent_deps += [ "//media/starboard" ] +} + if (is_fuchsia) { media_subcomponent_deps += [ "//media/fuchsia/common" ] } diff --git a/media/starboard/BUILD.gn b/media/starboard/BUILD.gn new file mode 100644 index 000000000000..774d7f7a7298 --- /dev/null +++ b/media/starboard/BUILD.gn @@ -0,0 +1,84 @@ +# Copyright 2024 The Cobalt Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source_set("starboard") { + visibility = [ "//media" ] + + defines = [ + "STARBOARD", + "COBALT_MEDIA_ENABLE_BACKGROUND_SUPPORT=0", + + # TODO: Revisit CValStats support + "COBALT_MEDIA_ENABLE_CVAL=0", + "COBALT_MEDIA_ENABLE_DECODE_BUFFER_BUDGET=0", + + # TODO: Revisit DecodeTarget support + "COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER=0", + "COBALT_MEDIA_ENABLE_FORMAT_SUPPORT_QUERY_METRICS=0", + "COBALT_MEDIA_ENABLE_IAMF_SUPPORT=0", + "COBALT_MEDIA_ENABLE_ENHANCED_IS_TYPE_SUPPORTED=0", + + # TODO: Revisit max video input size + "COBALT_MEDIA_ENABLE_PLAYER_SET_MAX_VIDEO_INPUT_SIZE=0", + "COBALT_MEDIA_ENABLE_SIDE_DATA=0", + + # TODO: Revisit Statistics + "COBALT_MEDIA_ENABLE_STATISTICS=0", + + # TODO: Revisit suspend/resume support + "COBALT_MEDIA_ENABLE_SUSPEND_RESUME=0", + "COBALT_MEDIA_ENABLE_TRACE_EVENT=0", + ] + + sources = [ + "sbplayer_bridge.cc", + "sbplayer_bridge.h", + "sbplayer_interface.cc", + "sbplayer_interface.h", + "sbplayer_set_bounds_helper.cc", + "sbplayer_set_bounds_helper.h", + "starboard_renderer.cc", + "starboard_renderer.h", + "starboard_utils.cc", + "starboard_utils.h", + ] + + deps = [ + "//base", + "//media:media_buildflags", + "//media/base", + "//starboard:", + "//starboard/common", + ] + + configs += [ + "//media:subcomponent_config", + "//starboard/build/config:size", + ] +} + +source_set("unit_tests") { + testonly = true + sources = [ "starboard_utils_test.cc" ] + configs += [ "//media:media_config" ] + deps = [ + "//base", + "//base/test:test_support", + "//media:test_support", + "//starboard:", + "//starboard/common", + "//testing/gmock", + "//testing/gtest", + ] +} diff --git a/media/starboard/sbplayer_bridge.cc b/media/starboard/sbplayer_bridge.cc index 44173baab20a..df9a15ec3bb5 100644 --- a/media/starboard/sbplayer_bridge.cc +++ b/media/starboard/sbplayer_bridge.cc @@ -12,30 +12,33 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "cobalt/media/base/sbplayer_bridge.h" +#include "media/starboard/sbplayer_bridge.h" #include #include #include #include -#include "base/bind.h" #include "base/compiler_specific.h" +#include "base/functional/bind.h" #include "base/location.h" #include "base/logging.h" #include "base/metrics/histogram_functions.h" #include "base/trace_event/trace_event.h" +#if COBALT_MEDIA_ENABLE_STATISTICS #include "cobalt/base/statistics.h" #include "cobalt/media/base/format_support_query_metrics.h" -#include "media/base/starboard_utils.h" +#endif // COBALT_MEDIA_ENABLE_STATISTICS +#include "media/starboard/starboard_utils.h" #include "starboard/common/media.h" #include "starboard/common/once.h" #include "starboard/common/player.h" #include "starboard/common/string.h" #include "starboard/configuration.h" +#if COBALT_MEDIA_ENABLE_PLAYER_SET_MAX_VIDEO_INPUT_SIZE #include "starboard/extension/player_set_max_video_input_size.h" +#endif // COBALT_MEDIA_ENABLE_PLAYER_SET_MAX_VIDEO_INPUT_SIZE -namespace cobalt { namespace media { namespace { @@ -45,28 +48,14 @@ using base::TimeDelta; using starboard::FormatString; using starboard::GetPlayerOutputModeName; +#if COBALT_MEDIA_ENABLE_STATISTICS class StatisticsWrapper { public: static StatisticsWrapper* GetInstance(); base::Statistics startup_latency{ "Media.PlaybackStartupLatency"}; }; - -void SetStreamInfo( - const SbMediaAudioStreamInfo& stream_info, - CobaltExtensionEnhancedAudioMediaAudioSampleInfo* sample_info) { - DCHECK(sample_info); - - sample_info->stream_info.codec = stream_info.codec; - sample_info->stream_info.mime = stream_info.mime; - sample_info->stream_info.number_of_channels = stream_info.number_of_channels; - sample_info->stream_info.samples_per_second = stream_info.samples_per_second; - sample_info->stream_info.bits_per_sample = stream_info.bits_per_sample; - sample_info->stream_info.audio_specific_config_size = - stream_info.audio_specific_config_size; - sample_info->stream_info.audio_specific_config = - stream_info.audio_specific_config; -} +#endif // COBALT_MEDIA_ENABLE_STATISTICS void SetStreamInfo(const SbMediaAudioStreamInfo& stream_info, SbMediaAudioSampleInfo* sample_info) { @@ -75,36 +64,12 @@ void SetStreamInfo(const SbMediaAudioStreamInfo& stream_info, sample_info->stream_info = stream_info; } -void SetStreamInfo( - const SbMediaVideoStreamInfo& stream_info, - CobaltExtensionEnhancedAudioMediaVideoSampleInfo* sample_info) { - DCHECK(sample_info); - - sample_info->stream_info.codec = stream_info.codec; - sample_info->stream_info.mime = stream_info.mime; - sample_info->stream_info.max_video_capabilities = - stream_info.max_video_capabilities; - sample_info->stream_info.frame_width = stream_info.frame_width; - sample_info->stream_info.frame_height = stream_info.frame_height; -} - void SetStreamInfo(const SbMediaVideoStreamInfo& stream_info, SbMediaVideoSampleInfo* sample_info) { DCHECK(sample_info); sample_info->stream_info = stream_info; } -void SetDiscardPadding( - const ::media::DecoderBuffer::DiscardPadding& discard_padding, - CobaltExtensionEnhancedAudioMediaAudioSampleInfo* sample_info) { - DCHECK(sample_info); - - sample_info->discarded_duration_from_front = - discard_padding.first.InMicroseconds(); - sample_info->discarded_duration_from_back = - discard_padding.second.InMicroseconds(); -} - void SetDiscardPadding( const ::media::DecoderBuffer::DiscardPadding& discard_padding, SbMediaAudioSampleInfo* sample_info) { @@ -118,7 +83,9 @@ void SetDiscardPadding( } // namespace +#if COBALT_MEDIA_ENABLE_STATISTICS SB_ONCE_INITIALIZE_FUNCTION(StatisticsWrapper, StatisticsWrapper::GetInstance); +#endif // COBALT_MEDIA_ENABLE_STATISTICS SbPlayerBridge::CallbackHelper::CallbackHelper(SbPlayerBridge* player_bridge) : player_bridge_(player_bridge) {} @@ -130,7 +97,7 @@ void SbPlayerBridge::CallbackHelper::ClearDecoderBufferCache() { } } -void SbPlayerBridge::CallbackHelper::OnDecoderStatus(SbPlayer player, +void SbPlayerBridge::CallbackHelper::OnDecoderStatus(void* player, SbMediaType type, SbPlayerDecoderState state, int ticket) { @@ -141,21 +108,23 @@ void SbPlayerBridge::CallbackHelper::OnDecoderStatus(SbPlayer player, } } -void SbPlayerBridge::CallbackHelper::OnPlayerStatus(SbPlayer player, +void SbPlayerBridge::CallbackHelper::OnPlayerStatus(void* player, SbPlayerState state, int ticket) { base::AutoLock auto_lock(lock_); if (player_bridge_) { - player_bridge_->OnPlayerStatus(player, state, ticket); + player_bridge_->OnPlayerStatus(static_cast(player), state, + ticket); } } -void SbPlayerBridge::CallbackHelper::OnPlayerError(SbPlayer player, +void SbPlayerBridge::CallbackHelper::OnPlayerError(void* player, SbPlayerError error, const std::string& message) { base::AutoLock auto_lock(lock_); if (player_bridge_) { - player_bridge_->OnPlayerError(player, error, message); + player_bridge_->OnPlayerError(static_cast(player), error, + message); } } @@ -208,10 +177,12 @@ SbPlayerBridge::SbPlayerBridge( CreateUrlPlayer(url_); +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME task_runner->PostTask( - FROM_HERE, - base::Bind(&SbPlayerBridge::CallbackHelper::ClearDecoderBufferCache, - callback_helper_)); + FROM_HERE, base::BindRepeatedly( + &SbPlayerBridge::CallbackHelper::ClearDecoderBufferCache, + callback_helper_)); +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME } #endif // SB_HAS(PLAYER_WITH_URL) @@ -230,7 +201,9 @@ SbPlayerBridge::SbPlayerBridge( SbPlayerSetBoundsHelper* set_bounds_helper, bool allow_resume_after_suspend, SbPlayerOutputMode default_output_mode, +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER DecodeTargetProvider* const decode_target_provider, +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER const std::string& max_video_capabilities, int max_video_input_size, std::string pipeline_identifier) @@ -238,8 +211,7 @@ SbPlayerBridge::SbPlayerBridge( task_runner_(task_runner), get_decode_target_graphics_context_provider_func_( get_decode_target_graphics_context_provider_func), - callback_helper_( - new CallbackHelper(ALLOW_THIS_IN_INITIALIZER_LIST(this))), + callback_helper_(new CallbackHelper(this)), window_(window), drm_system_(drm_system), host_(host), @@ -247,21 +219,29 @@ SbPlayerBridge::SbPlayerBridge( allow_resume_after_suspend_(allow_resume_after_suspend), audio_config_(audio_config), video_config_(video_config), +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER decode_target_provider_(decode_target_provider), +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER max_video_capabilities_(max_video_capabilities), max_video_input_size_(max_video_input_size), +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_(&interface->cval_stats_), +#endif // COBALT_MEDIA_ENABLE_CVAL pipeline_identifier_(pipeline_identifier) #if SB_HAS(PLAYER_WITH_URL) , is_url_based_(false) #endif // SB_HAS(PLAYER_WITH_URL { +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER DCHECK(!get_decode_target_graphics_context_provider_func_.is_null()); +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER DCHECK(audio_config.IsValidConfig() || video_config.IsValidConfig()); DCHECK(host_); DCHECK(set_bounds_helper_); +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER DCHECK(decode_target_provider_); +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER audio_stream_info_.codec = kSbMediaAudioCodecNone; video_stream_info_.codec = kSbMediaVideoCodecNone; @@ -278,12 +258,14 @@ SbPlayerBridge::SbPlayerBridge( CreatePlayer(); +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME if (SbPlayerIsValid(player_)) { task_runner->PostTask( - FROM_HERE, - base::Bind(&SbPlayerBridge::CallbackHelper::ClearDecoderBufferCache, - callback_helper_)); + FROM_HERE, base::BindRepeatedly( + &SbPlayerBridge::CallbackHelper::ClearDecoderBufferCache, + callback_helper_)); } +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME } SbPlayerBridge::~SbPlayerBridge() { @@ -292,14 +274,20 @@ SbPlayerBridge::~SbPlayerBridge() { callback_helper_->ResetPlayer(); set_bounds_helper_->SetPlayerBridge(NULL); +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER decode_target_provider_->SetOutputMode( DecodeTargetProvider::kOutputModeInvalid); decode_target_provider_->ResetGetCurrentSbDecodeTargetFunction(); +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER if (SbPlayerIsValid(player_)) { +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StartTimer(MediaTiming::SbPlayerDestroy, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL sbplayer_interface_->Destroy(player_); +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StopTimer(MediaTiming::SbPlayerDestroy, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL } } @@ -350,6 +338,7 @@ void SbPlayerBridge::WriteBuffers( DCHECK(!is_url_based_); #endif // SB_HAS(PLAYER_WITH_URL) +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME if (allow_resume_after_suspend_) { if (type == DemuxerStream::Type::AUDIO) { decoder_buffer_cache_.AddBuffers(buffers, audio_stream_info_); @@ -362,14 +351,10 @@ void SbPlayerBridge::WriteBuffers( } return; } +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME - if (sbplayer_interface_->IsEnhancedAudioExtensionEnabled()) { - WriteBuffersInternal( - type, buffers, &audio_stream_info_, &video_stream_info_); - } else { - WriteBuffersInternal(type, buffers, &audio_stream_info_, - &video_stream_info_); - } + WriteBuffersInternal(type, buffers, &audio_stream_info_, + &video_stream_info_); } void SbPlayerBridge::SetBounds(int z_index, const gfx::Rect& rect) { @@ -401,7 +386,9 @@ void SbPlayerBridge::PrepareForSeek() { void SbPlayerBridge::Seek(TimeDelta time) { DCHECK(task_runner_->RunsTasksInCurrentSequence()); +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME decoder_buffer_cache_.ClearAll(); +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME seek_pending_ = false; pending_audio_eos_buffer_ = false; @@ -455,8 +442,8 @@ void SbPlayerBridge::SetPlaybackRate(double playback_rate) { sbplayer_interface_->SetPlaybackRate(player_, playback_rate); } -void SbPlayerBridge::GetInfo(uint32* video_frames_decoded, - uint32* video_frames_dropped, +void SbPlayerBridge::GetInfo(uint32_t* video_frames_decoded, + uint32_t* video_frames_dropped, TimeDelta* media_time) { DCHECK(video_frames_decoded || video_frames_dropped || media_time); @@ -604,13 +591,19 @@ void SbPlayerBridge::Suspend() { state_ = kSuspended; +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER decode_target_provider_->SetOutputMode( DecodeTargetProvider::kOutputModeInvalid); decode_target_provider_->ResetGetCurrentSbDecodeTargetFunction(); +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StartTimer(MediaTiming::SbPlayerDestroy, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL sbplayer_interface_->Destroy(player_); +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StopTimer(MediaTiming::SbPlayerDestroy, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL player_ = kSbPlayerInvalid; } @@ -626,7 +619,9 @@ void SbPlayerBridge::Resume(SbWindow window) { return; } +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME decoder_buffer_cache_.StartResuming(); +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME #if SB_HAS(PLAYER_WITH_URL) if (is_url_based_) { @@ -648,7 +643,10 @@ void SbPlayerBridge::Resume(SbWindow window) { } } +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER + namespace { + DecodeTargetProvider::OutputMode ToVideoFrameProviderOutputMode( SbPlayerOutputMode output_mode) { switch (output_mode) { @@ -666,6 +664,8 @@ DecodeTargetProvider::OutputMode ToVideoFrameProviderOutputMode( } // namespace +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER + #if SB_HAS(PLAYER_WITH_URL) // static void SbPlayerBridge::EncryptedMediaInitDataEncounteredCB( @@ -721,12 +721,18 @@ void SbPlayerBridge::CreateUrlPlayer(const std::string& url) { UpdateBounds_Locked(); } #endif // SB_HAS(PLAYER_WITH_URL) + void SbPlayerBridge::CreatePlayer() { +#if COBALT_MEDIA_ENABLE_TRACE_EVENT TRACE_EVENT0("cobalt::media", "SbPlayerBridge::CreatePlayer"); +#endif // COBALT_MEDIA_ENABLE_TRACE_EVENT DCHECK(task_runner_->RunsTasksInCurrentSequence()); +#if COBALT_MEDIA_ENABLE_BACKGROUND_SUPPORT bool is_visible = SbWindowIsValid(window_); - bool has_audio = audio_stream_info_.codec != kSbMediaAudioCodecNone; +#else // COBALT_MEDIA_ENABLE_BACKGROUND_SUPPORT + bool is_visible = true; +#endif // COBALT_MEDIA_ENABLE_BACKGROUND_SUPPORT is_creating_player_ = true; @@ -739,9 +745,11 @@ void SbPlayerBridge::CreatePlayer() { return; } +#if COBALT_MEDIA_ENABLE_FORMAT_SUPPORT_QUERY_METRICS if (max_video_capabilities_.empty()) { FormatSupportQueryMetrics::PrintAndResetMetrics(); } +#endif // COBALT_MEDIA_ENABLE_FORMAT_SUPPORT_QUERY_METRICS player_creation_time_ = Time::Now(); @@ -758,7 +766,10 @@ void SbPlayerBridge::CreatePlayer() { creation_param.output_mode = output_mode_; DCHECK_EQ(sbplayer_interface_->GetPreferredOutputMode(&creation_param), output_mode_); +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StartTimer(MediaTiming::SbPlayerCreate, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL +#if COBALT_MEDIA_ENABLE_PLAYER_SET_MAX_VIDEO_INPUT_SIZE const StarboardExtensionPlayerSetMaxVideoInputSizeApi* player_set_max_video_input_size_extension = static_cast( @@ -771,12 +782,19 @@ void SbPlayerBridge::CreatePlayer() { player_set_max_video_input_size_extension ->SetMaxVideoInputSizeForCurrentThread(max_video_input_size_); } +#endif // COBALT_MEDIA_ENABLE_PLAYER_SET_MAX_VIDEO_INPUT_SIZE player_ = sbplayer_interface_->Create( window_, &creation_param, &SbPlayerBridge::DeallocateSampleCB, &SbPlayerBridge::DecoderStatusCB, &SbPlayerBridge::PlayerStatusCB, &SbPlayerBridge::PlayerErrorCB, this, +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER get_decode_target_graphics_context_provider_func_.Run()); +#else // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER + nullptr); +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StopTimer(MediaTiming::SbPlayerCreate, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL is_creating_player_ = false; @@ -787,21 +805,26 @@ void SbPlayerBridge::CreatePlayer() { if (output_mode_ == kSbPlayerOutputModeDecodeToTexture) { // If the player is setup to decode to texture, then provide Cobalt with // a method of querying that texture. +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER decode_target_provider_->SetGetCurrentSbDecodeTargetFunction(base::Bind( &SbPlayerBridge::GetCurrentSbDecodeTarget, base::Unretained(this))); +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER LOG(INFO) << "Playing in decode-to-texture mode."; } else { LOG(INFO) << "Playing in punch-out mode."; } +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER decode_target_provider_->SetOutputMode( ToVideoFrameProviderOutputMode(output_mode_)); +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER set_bounds_helper_->SetPlayerBridge(this); base::AutoLock auto_lock(lock_); UpdateBounds_Locked(); } +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME void SbPlayerBridge::WriteNextBuffersFromCache(DemuxerStream::Type type, int max_buffers_per_write) { DCHECK(state_ != kSuspended); @@ -842,6 +865,7 @@ void SbPlayerBridge::WriteNextBuffersFromCache(DemuxerStream::Type type, } } } +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME template void SbPlayerBridge::WriteBuffersInternal( @@ -871,10 +895,10 @@ void SbPlayerBridge::WriteBuffersInternal( gathered_sbplayer_sample_infos_subsample_mapping.reserve(buffers.size()); gathered_sbplayer_sample_infos_side_data.reserve(buffers.size()); - for (int i = 0; i < buffers.size(); i++) { + for (int i = 0; i < static_cast(buffers.size()); i++) { const auto& buffer = buffers[i]; if (buffer->end_of_stream()) { - DCHECK_EQ(i, buffers.size() - 1); + DCHECK_EQ(static_cast(i), buffers.size() - 1); if (type == DemuxerStream::AUDIO) { pending_audio_eos_buffer_ = true; } else { @@ -913,8 +937,10 @@ void SbPlayerBridge::WriteBuffersInternal( gathered_sbplayer_sample_infos_side_data.push_back( SbPlayerSampleSideData()); +#if COBALT_MEDIA_ENABLE_SIDE_DATA SbPlayerSampleSideData* side_data = &gathered_sbplayer_sample_infos_side_data[i]; +#endif // COBALT_MEDIA_ENABLE_SIDE_DATA PlayerSampleInfo sample_info = {}; sample_info.type = sample_type; @@ -922,6 +948,7 @@ void SbPlayerBridge::WriteBuffersInternal( sample_info.buffer_size = buffer->data_size(); sample_info.timestamp = buffer->timestamp().InMicroseconds(); +#if COBALT_MEDIA_ENABLE_SIDE_DATA if (buffer->side_data_size() > 0) { // We only support at most one side data currently. side_data->type = kMatroskaBlockAdditional; @@ -930,6 +957,7 @@ void SbPlayerBridge::WriteBuffersInternal( sample_info.side_data = side_data; sample_info.side_data_count = 1; } +#endif // COBALT_MEDIA_ENABLE_SIDE_DATA if (sample_type == kSbMediaTypeAudio) { DCHECK(audio_stream_info); @@ -951,13 +979,17 @@ void SbPlayerBridge::WriteBuffersInternal( } if (!gathered_sbplayer_sample_infos.empty()) { +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StartTimer(MediaTiming::SbPlayerWriteSamples, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL sbplayer_interface_->WriteSamples(player_, sample_type, gathered_sbplayer_sample_infos.data(), gathered_sbplayer_sample_infos.size()); +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StopTimer(MediaTiming::SbPlayerWriteSamples, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL } } @@ -969,8 +1001,8 @@ SbPlayerOutputMode SbPlayerBridge::GetSbPlayerOutputMode() { return output_mode_; } -void SbPlayerBridge::GetInfo_Locked(uint32* video_frames_decoded, - uint32* video_frames_dropped, +void SbPlayerBridge::GetInfo_Locked(uint32_t* video_frames_decoded, + uint32_t* video_frames_dropped, TimeDelta* media_time) { lock_.AssertAcquired(); if (state_ == kSuspended) { @@ -992,7 +1024,7 @@ void SbPlayerBridge::GetInfo_Locked(uint32* video_frames_decoded, sbplayer_interface_->GetInfo(player_, &info); if (media_time) { - *media_time = TimeDelta::FromMicroseconds(info.current_media_timestamp); + *media_time = base::Microseconds(info.current_media_timestamp); } if (video_frames_decoded) { *video_frames_decoded = info.total_video_frames; @@ -1018,6 +1050,7 @@ void SbPlayerBridge::UpdateBounds_Locked() { void SbPlayerBridge::ClearDecoderBufferCache() { DCHECK(task_runner_->RunsTasksInCurrentSequence()); +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME if (state_ != kResuming) { TimeDelta media_time; GetInfo(NULL, NULL, &media_time); @@ -1029,6 +1062,7 @@ void SbPlayerBridge::ClearDecoderBufferCache() { base::Bind(&SbPlayerBridge::CallbackHelper::ClearDecoderBufferCache, callback_helper_), TimeDelta::FromMilliseconds(kClearDecoderCacheIntervalInMilliseconds)); +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME } void SbPlayerBridge::OnDecoderStatus(SbPlayer player, @@ -1067,6 +1101,7 @@ void SbPlayerBridge::OnDecoderStatus(SbPlayer player, auto max_number_of_samples_to_write = SbPlayerGetMaximumNumberOfSamplesPerWrite(player_, type); if (state_ == kResuming) { +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME if (decoder_buffer_cache_.HasMoreBuffers(stream_type)) { WriteNextBuffersFromCache(stream_type, max_number_of_samples_to_write); return; @@ -1075,6 +1110,9 @@ void SbPlayerBridge::OnDecoderStatus(SbPlayer player, !decoder_buffer_cache_.HasMoreBuffers(DemuxerStream::VIDEO)) { state_ = kPlaying; } +#else // COBALT_MEDIA_ENABLE_SUSPEND_RESUME + state_ = kPlaying; +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME } host_->OnNeedData(stream_type, max_number_of_samples_to_write); } @@ -1082,8 +1120,10 @@ void SbPlayerBridge::OnDecoderStatus(SbPlayer player, void SbPlayerBridge::OnPlayerStatus(SbPlayer player, SbPlayerState state, int ticket) { +#if COBALT_MEDIA_ENABLE_TRACE_EVENT TRACE_EVENT1("cobalt::media", "SbPlayerBridge::OnPlayerStatus", "state", state); +#endif // COBALT_MEDIA_ENABLE_TRACE_EVENT DCHECK(task_runner_->RunsTasksInCurrentSequence()); if (player_ != player) { @@ -1173,8 +1213,9 @@ void SbPlayerBridge::DecoderStatusCB(SbPlayer player, SbPlayerBridge* helper = static_cast(context); helper->task_runner_->PostTask( FROM_HERE, - base::Bind(&SbPlayerBridge::CallbackHelper::OnDecoderStatus, - helper->callback_helper_, player, type, state, ticket)); + base::BindOnce(&SbPlayerBridge::CallbackHelper::OnDecoderStatus, + helper->callback_helper_, static_cast(player), type, + state, ticket)); } // static @@ -1184,8 +1225,9 @@ void SbPlayerBridge::PlayerStatusCB(SbPlayer player, int ticket) { SbPlayerBridge* helper = static_cast(context); helper->task_runner_->PostTask( - FROM_HERE, base::Bind(&SbPlayerBridge::CallbackHelper::OnPlayerStatus, - helper->callback_helper_, player, state, ticket)); + FROM_HERE, base::BindOnce(&SbPlayerBridge::CallbackHelper::OnPlayerStatus, + helper->callback_helper_, + static_cast(player), state, ticket)); } // static @@ -1202,9 +1244,10 @@ void SbPlayerBridge::PlayerErrorCB(SbPlayer player, } } helper->task_runner_->PostTask( - FROM_HERE, base::Bind(&SbPlayerBridge::CallbackHelper::OnPlayerError, - helper->callback_helper_, player, error, - message ? std::string(message) : "")); + FROM_HERE, + base::BindOnce(&SbPlayerBridge::CallbackHelper::OnPlayerError, + helper->callback_helper_, static_cast(player), + error, message ? std::string(message) : "")); } // static @@ -1213,8 +1256,9 @@ void SbPlayerBridge::DeallocateSampleCB(SbPlayer player, const void* sample_buffer) { SbPlayerBridge* helper = static_cast(context); helper->task_runner_->PostTask( - FROM_HERE, base::Bind(&SbPlayerBridge::CallbackHelper::OnDeallocateSample, - helper->callback_helper_, sample_buffer)); + FROM_HERE, + base::BindOnce(&SbPlayerBridge::CallbackHelper::OnDeallocateSample, + helper->callback_helper_, sample_buffer)); } #if SB_HAS(PLAYER_WITH_URL) @@ -1316,8 +1360,10 @@ void SbPlayerBridge::LogStartupLatency() const { TimeDelta startup_latency = sb_player_state_presenting_time_ - first_event_time; +#if COBALT_MEDIA_ENABLE_STATISTICS StatisticsWrapper::GetInstance()->startup_latency.AddSample( startup_latency.InMicroseconds(), 1); +#endif // COBALT_MEDIA_ENABLE_STATISTICS // clang-format off LOG(INFO) << FormatString( @@ -1335,10 +1381,14 @@ void SbPlayerBridge::LogStartupLatency() const { first_events_str.c_str(), player_initialization_time_delta.InMicroseconds(), player_preroll_time_delta.InMicroseconds(), first_audio_sample_time_delta.InMicroseconds(), first_video_sample_time_delta.InMicroseconds(), player_presenting_time_delta.InMicroseconds(), +#if COBALT_MEDIA_ENABLE_STATISTICS StatisticsWrapper::GetInstance()->startup_latency.min(), StatisticsWrapper::GetInstance()->startup_latency.GetMedian(), StatisticsWrapper::GetInstance()->startup_latency.average(), StatisticsWrapper::GetInstance()->startup_latency.max()); +#else // COBALT_MEDIA_ENABLE_STATISTICS + 0ll, 0ll, 0ll, 0ll); +#endif // COBALT_MEDIA_ENABLE_STATISTICS // clang-format on } @@ -1361,4 +1411,3 @@ void SbPlayerBridge::SendColorSpaceHistogram() const { } } // namespace media -} // namespace cobalt diff --git a/media/starboard/sbplayer_bridge.h b/media/starboard/sbplayer_bridge.h index c361fea3ff04..c1709493eb38 100644 --- a/media/starboard/sbplayer_bridge.h +++ b/media/starboard/sbplayer_bridge.h @@ -12,42 +12,47 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef COBALT_MEDIA_BASE_SBPLAYER_BRIDGE_H_ -#define COBALT_MEDIA_BASE_SBPLAYER_BRIDGE_H_ +#ifndef MEDIA_STARBOARD_SBPLAYER_BRIDGE_H_ +#define MEDIA_STARBOARD_SBPLAYER_BRIDGE_H_ #include #include #include #include +#include "base/functional/callback.h" #include "base/memory/ref_counted.h" -#include "base/single_thread_task_runner.h" #include "base/synchronization/lock.h" #include "base/task/sequenced_task_runner.h" +#include "base/task/single_thread_task_runner.h" #include "base/time/time.h" + +#if COBALT_MEDIA_ENABLE_CVAL #include "cobalt/media/base/cval_stats.h" +#endif // COBALT_MEDIA_ENABLE_CVAL + +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER #include "cobalt/media/base/decode_target_provider.h" +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER + +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME #include "cobalt/media/base/decoder_buffer_cache.h" -#include "cobalt/media/base/sbplayer_interface.h" -#include "cobalt/media/base/sbplayer_set_bounds_helper.h" +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME + #include "media/base/audio_decoder_config.h" #include "media/base/decoder_buffer.h" #include "media/base/demuxer_stream.h" #include "media/base/video_decoder_config.h" +#include "media/starboard/sbplayer_interface.h" +#include "media/starboard/sbplayer_set_bounds_helper.h" #include "starboard/media.h" #include "starboard/player.h" #include "ui/gfx/geometry/rect.h" -namespace cobalt { namespace media { // TODO: Add switch to disable caching class SbPlayerBridge { - typedef ::media::AudioDecoderConfig AudioDecoderConfig; - typedef ::media::DecoderBuffer DecoderBuffer; - typedef ::media::DemuxerStream DemuxerStream; - typedef ::media::VideoDecoderConfig VideoDecoderConfig; - public: class Host { public: @@ -62,7 +67,7 @@ class SbPlayerBridge { }; // Call to get the SbDecodeTargetGraphicsContextProvider for SbPlayerCreate(). - typedef base::Callback + typedef base::RepeatingCallback GetDecodeTargetGraphicsContextProviderFunc; #if SB_HAS(PLAYER_WITH_URL) @@ -97,7 +102,9 @@ class SbPlayerBridge { SbPlayerSetBoundsHelper* set_bounds_helper, bool allow_resume_after_suspend, SbPlayerOutputMode default_output_mode, +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER DecodeTargetProvider* const decode_target_provider, +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER const std::string& max_video_capabilities, int max_video_input_size, std::string pipeline_identifier); @@ -121,8 +128,8 @@ class SbPlayerBridge { void SetVolume(float volume); void SetPlaybackRate(double playback_rate); - void GetInfo(uint32* video_frames_decoded, - uint32* video_frames_dropped, + void GetInfo(uint32_t* video_frames_decoded, + uint32_t* video_frames_dropped, base::TimeDelta* media_time); std::vector GetAudioConfigurations(); @@ -171,12 +178,14 @@ class SbPlayerBridge { void ClearDecoderBufferCache(); - void OnDecoderStatus(SbPlayer player, + // The following functions accept SbPlayer as void* to work around the + // requirements that types binding to Callbacks have to be complete. + void OnDecoderStatus(void* player, SbMediaType type, SbPlayerDecoderState state, int ticket); - void OnPlayerStatus(SbPlayer player, SbPlayerState state, int ticket); - void OnPlayerError(SbPlayer player, + void OnPlayerStatus(void* player, SbPlayerState state, int ticket); + void OnPlayerError(void* player, SbPlayerError error, const std::string& message); void OnDeallocateSample(const void* sample_buffer); @@ -188,7 +197,7 @@ class SbPlayerBridge { SbPlayerBridge* player_bridge_; }; - static const int64 kClearDecoderCacheIntervalInMilliseconds = 1000; + static const int64_t kClearDecoderCacheIntervalInMilliseconds = 1000; // A map from raw data pointer returned by DecoderBuffer::GetData() to the // DecoderBuffer and a reference count. The reference count indicates how @@ -212,8 +221,10 @@ class SbPlayerBridge { #endif // SB_HAS(PLAYER_WITH_URL) void CreatePlayer(); +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME void WriteNextBuffersFromCache(DemuxerStream::Type type, int max_buffers_per_write); +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME template void WriteBuffersInternal( @@ -222,8 +233,8 @@ class SbPlayerBridge { const SbMediaAudioStreamInfo* audio_stream_info, const SbMediaVideoStreamInfo* video_stream_info); - void GetInfo_Locked(uint32* video_frames_decoded, - uint32* video_frames_dropped, + void GetInfo_Locked(uint32_t* video_frames_decoded, + uint32_t* video_frames_dropped, base::TimeDelta* media_time); void UpdateBounds_Locked(); @@ -297,25 +308,29 @@ class SbPlayerBridge { float volume_ = 1.0f; double playback_rate_ = 0.0; bool seek_pending_ = false; +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME DecoderBufferCache decoder_buffer_cache_; +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME // The following variables can be accessed from GetInfo(), which can be called // from any threads. So some of their usages have to be guarded by |lock_|. base::Lock lock_; // Stores the |z_index| and |rect| parameters of the latest SetBounds() call. - base::Optional set_bounds_z_index_; - base::Optional set_bounds_rect_; + std::optional set_bounds_z_index_; + std::optional set_bounds_rect_; State state_ = kPlaying; SbPlayer player_; - uint32 cached_video_frames_decoded_; - uint32 cached_video_frames_dropped_; + uint32_t cached_video_frames_decoded_; + uint32_t cached_video_frames_dropped_; base::TimeDelta preroll_timestamp_; // Keep track of the output mode we are supposed to output to. SbPlayerOutputMode output_mode_; +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER DecodeTargetProvider* const decode_target_provider_; +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER // Keep copies of the mime type strings instead of using the ones in the // DemuxerStreams to ensure that the strings are always valid. @@ -348,11 +363,12 @@ class SbPlayerBridge { bool pending_audio_eos_buffer_ = false; bool pending_video_eos_buffer_ = false; +#if COBALT_MEDIA_ENABLE_CVAL CValStats* cval_stats_; +#endif // COBALT_MEDIA_ENABLE_CVAL std::string pipeline_identifier_; }; } // namespace media -} // namespace cobalt -#endif // COBALT_MEDIA_BASE_SBPLAYER_BRIDGE_H_ +#endif // MEDIA_STARBOARD_SBPLAYER_BRIDGE_H_ diff --git a/media/starboard/sbplayer_interface.cc b/media/starboard/sbplayer_interface.cc index 8db489719785..74f9b9313334 100644 --- a/media/starboard/sbplayer_interface.cc +++ b/media/starboard/sbplayer_interface.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "cobalt/media/base/sbplayer_interface.h" +#include "media/starboard/sbplayer_interface.h" #include @@ -20,7 +20,6 @@ #include "starboard/extension/player_configuration.h" #include "starboard/system.h" -namespace cobalt { namespace media { bool SbPlayerInterface::SetDecodeToTexturePreferred(bool preferred) { @@ -46,23 +45,6 @@ bool SbPlayerInterface::SetDecodeToTexturePreferred(bool preferred) { } } -DefaultSbPlayerInterface::DefaultSbPlayerInterface() { - const CobaltExtensionEnhancedAudioApi* extension_api = - static_cast( - SbSystemGetExtension(kCobaltExtensionEnhancedAudioName)); - if (!extension_api) { - return; - } - - DCHECK_EQ(extension_api->name, - // Avoid comparing raw string pointers for equal. - std::string(kCobaltExtensionEnhancedAudioName)); - DCHECK_EQ(extension_api->version, 1u); - DCHECK_NE(extension_api->PlayerWriteSamples, nullptr); - - enhanced_audio_player_write_samples_ = extension_api->PlayerWriteSamples; -} - SbPlayer DefaultSbPlayerInterface::Create( SbWindow window, const SbPlayerCreationParam* creation_param, @@ -104,30 +86,15 @@ void DefaultSbPlayerInterface::Seek(SbPlayer player, media_metrics_provider_.EndTrackingAction(MediaAction::SBPLAYER_SEEK); } -bool DefaultSbPlayerInterface::IsEnhancedAudioExtensionEnabled() const { - return enhanced_audio_player_write_samples_ != nullptr; -} - void DefaultSbPlayerInterface::WriteSamples( SbPlayer player, SbMediaType sample_type, const SbPlayerSampleInfo* sample_infos, int number_of_sample_infos) { - DCHECK(!IsEnhancedAudioExtensionEnabled()); SbPlayerWriteSamples(player, sample_type, sample_infos, number_of_sample_infos); } -void DefaultSbPlayerInterface::WriteSamples( - SbPlayer player, - SbMediaType sample_type, - const CobaltExtensionEnhancedAudioPlayerSampleInfo* sample_infos, - int number_of_sample_infos) { - DCHECK(IsEnhancedAudioExtensionEnabled()); - enhanced_audio_player_write_samples_(player, sample_type, sample_infos, - number_of_sample_infos); -} - int DefaultSbPlayerInterface::GetMaximumNumberOfSamplesPerWrite( SbPlayer player, SbMediaType sample_type) { @@ -237,4 +204,3 @@ bool DefaultSbPlayerInterface::GetAudioConfiguration( } } // namespace media -} // namespace cobalt diff --git a/media/starboard/sbplayer_interface.h b/media/starboard/sbplayer_interface.h index d29d1c31a9c8..f0085ee91207 100644 --- a/media/starboard/sbplayer_interface.h +++ b/media/starboard/sbplayer_interface.h @@ -12,20 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef COBALT_MEDIA_BASE_SBPLAYER_INTERFACE_H_ -#define COBALT_MEDIA_BASE_SBPLAYER_INTERFACE_H_ +#ifndef MEDIA_STARBOARD_SBPLAYER_INTERFACE_H_ +#define MEDIA_STARBOARD_SBPLAYER_INTERFACE_H_ #include "base/time/time.h" +#if COBALT_MEDIA_ENABLE_CVAL #include "cobalt/media/base/cval_stats.h" +#endif // COBALT_MEDIA_ENABLE_CVAL +#if COBALT_MEDIA_ENABLE_STATISTICS #include "cobalt/media/base/metrics_provider.h" -#include "starboard/extension/enhanced_audio.h" +#endif // COBALT_MEDIA_ENABLE_STATISTICS #include "starboard/player.h" #if SB_HAS(PLAYER_WITH_URL) #include SB_URL_PLAYER_INCLUDE_PATH #endif // SB_HAS(PLAYER_WITH_URL) -namespace cobalt { namespace media { class SbPlayerInterface { @@ -48,16 +50,10 @@ class SbPlayerInterface { base::TimeDelta seek_to_timestamp, int ticket) = 0; - virtual bool IsEnhancedAudioExtensionEnabled() const = 0; virtual void WriteSamples(SbPlayer player, SbMediaType sample_type, const SbPlayerSampleInfo* sample_infos, int number_of_sample_infos) = 0; - virtual void WriteSamples( - SbPlayer player, - SbMediaType sample_type, - const CobaltExtensionEnhancedAudioPlayerSampleInfo* sample_infos, - int number_of_sample_infos) = 0; virtual int GetMaximumNumberOfSamplesPerWrite(SbPlayer player, SbMediaType sample_type) = 0; @@ -97,11 +93,42 @@ class SbPlayerInterface { int index, SbMediaAudioConfiguration* out_audio_configuration) = 0; +#if COBALT_MEDIA_ENABLE_CVAL // disabled by default, but can be enabled via h5vcc setting. void EnableCValStats(bool should_enable) { cval_stats_.Enable(should_enable); } CValStats cval_stats_; +#endif // COBALT_MEDIA_ENABLE_CVAL + +#if !COBALT_MEDIA_ENABLE_STATISTICS + enum class MediaAction { + UNKNOWN_ACTION, + WEBMEDIAPLAYER_SEEK, + SBPLAYER_CREATE, + SBPLAYER_CREATE_URL_PLAYER, + SBPLAYER_DESTROY, + SBPLAYER_GET_PREFERRED_OUTPUT_MODE, + SBPLAYER_SEEK, + SBPLAYER_WRITE_END_OF_STREAM_AUDIO, + SBPLAYER_WRITE_END_OF_STREAM_VIDEO, + SBPLAYER_SET_BOUNDS, + SBPLAYER_SET_PLAYBACK_RATE, + SBPLAYER_SET_VOLUME, + SBPLAYER_GET_INFO, + SBPLAYER_GET_CURRENT_FRAME, + SBPLAYER_GET_AUDIO_CONFIG, + SBDRM_CREATE, + SBDRM_DESTROY, + SBDRM_GENERATE_SESSION_UPDATE_REQUEST, + SBDRM_UPDATE_SESSION, + SBDRM_CLOSE_SESSION, + }; + struct MediaMetricsProvider { + void StartTrackingAction(...) {} + void EndTrackingAction(...) {} + }; +#endif // !COBALT_MEDIA_ENABLE_STATISTICS MediaMetricsProvider media_metrics_provider_; bool SetDecodeToTexturePreferred(bool preferred); @@ -109,8 +136,6 @@ class SbPlayerInterface { class DefaultSbPlayerInterface final : public SbPlayerInterface { public: - DefaultSbPlayerInterface(); - SbPlayer Create( SbWindow window, const SbPlayerCreationParam* creation_param, @@ -126,16 +151,10 @@ class DefaultSbPlayerInterface final : public SbPlayerInterface { void Seek(SbPlayer player, base::TimeDelta seek_to_timestamp, int ticket) override; - bool IsEnhancedAudioExtensionEnabled() const override; void WriteSamples(SbPlayer player, SbMediaType sample_type, const SbPlayerSampleInfo* sample_infos, int number_of_sample_infos) override; - void WriteSamples( - SbPlayer player, - SbMediaType sample_type, - const CobaltExtensionEnhancedAudioPlayerSampleInfo* sample_infos, - int number_of_sample_infos) override; int GetMaximumNumberOfSamplesPerWrite(SbPlayer player, SbMediaType sample_type) override; void WriteEndOfStream(SbPlayer player, SbMediaType stream_type) override; @@ -169,16 +188,8 @@ class DefaultSbPlayerInterface final : public SbPlayerInterface { SbPlayer player, int index, SbMediaAudioConfiguration* out_audio_configuration) override; - - private: - void (*enhanced_audio_player_write_samples_)( - SbPlayer player, - SbMediaType sample_type, - const CobaltExtensionEnhancedAudioPlayerSampleInfo* sample_infos, - int number_of_sample_infos) = nullptr; }; } // namespace media -} // namespace cobalt -#endif // COBALT_MEDIA_BASE_SBPLAYER_INTERFACE_H_ +#endif // MEDIA_STARBOARD_SBPLAYER_INTERFACE_H_ diff --git a/media/starboard/sbplayer_set_bounds_helper.cc b/media/starboard/sbplayer_set_bounds_helper.cc index 9b02952baf72..e49868f946f6 100644 --- a/media/starboard/sbplayer_set_bounds_helper.cc +++ b/media/starboard/sbplayer_set_bounds_helper.cc @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "cobalt/media/base/sbplayer_set_bounds_helper.h" +#include "media/starboard/sbplayer_set_bounds_helper.h" #include "base/atomic_sequence_num.h" -#include "cobalt/media/base/sbplayer_bridge.h" +#include "media/starboard/sbplayer_bridge.h" -namespace cobalt { namespace media { namespace { @@ -47,4 +46,3 @@ bool SbPlayerSetBoundsHelper::SetBounds(int x, int y, int width, int height) { } } // namespace media -} // namespace cobalt diff --git a/media/starboard/sbplayer_set_bounds_helper.h b/media/starboard/sbplayer_set_bounds_helper.h index 4467755038ff..c205d277e917 100644 --- a/media/starboard/sbplayer_set_bounds_helper.h +++ b/media/starboard/sbplayer_set_bounds_helper.h @@ -12,16 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef COBALT_MEDIA_BASE_SBPLAYER_SET_BOUNDS_HELPER_H_ -#define COBALT_MEDIA_BASE_SBPLAYER_SET_BOUNDS_HELPER_H_ +#ifndef MEDIA_STARBOARD_SBPLAYER_SET_BOUNDS_HELPER_H_ +#define MEDIA_STARBOARD_SBPLAYER_SET_BOUNDS_HELPER_H_ + +#include -#include "base/macros.h" #include "base/memory/ref_counted.h" -#include "base/optional.h" #include "base/synchronization/lock.h" #include "ui/gfx/geometry/rect.h" -namespace cobalt { namespace media { class SbPlayerBridge; @@ -37,12 +36,12 @@ class SbPlayerSetBoundsHelper private: base::Lock lock_; SbPlayerBridge* player_bridge_ = nullptr; - base::Optional rect_; + std::optional rect_; - DISALLOW_COPY_AND_ASSIGN(SbPlayerSetBoundsHelper); + SbPlayerSetBoundsHelper(const SbPlayerSetBoundsHelper&) = delete; + void operator=(const SbPlayerSetBoundsHelper&) = delete; }; } // namespace media -} // namespace cobalt -#endif // COBALT_MEDIA_BASE_SBPLAYER_SET_BOUNDS_HELPER_H_ +#endif // MEDIA_STARBOARD_SBPLAYER_SET_BOUNDS_HELPER_H_ diff --git a/media/starboard/starboard_renderer.cc b/media/starboard/starboard_renderer.cc new file mode 100644 index 000000000000..4b4f7008074a --- /dev/null +++ b/media/starboard/starboard_renderer.cc @@ -0,0 +1,573 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "media/starboard/starboard_renderer.h" + +#include "base/logging.h" +#include "starboard/common/media.h" + +namespace media { + +namespace { + +bool HasRemoteAudioOutputs( + const std::vector& configurations) { + for (auto&& configuration : configurations) { + const auto connector = configuration.connector; + switch (connector) { + case kSbMediaAudioConnectorUnknown: + case kSbMediaAudioConnectorAnalog: + case kSbMediaAudioConnectorBuiltIn: + case kSbMediaAudioConnectorHdmi: + case kSbMediaAudioConnectorSpdif: + case kSbMediaAudioConnectorUsb: + LOG(INFO) << "Encountered local audio connector: " + << starboard::GetMediaAudioConnectorName(connector); + break; + case kSbMediaAudioConnectorBluetooth: + case kSbMediaAudioConnectorRemoteWired: + case kSbMediaAudioConnectorRemoteWireless: + case kSbMediaAudioConnectorRemoteOther: + LOG(INFO) << "Encountered remote audio connector: " + << starboard::GetMediaAudioConnectorName(connector); + return true; + } + } + + LOG(INFO) << "No remote audio outputs found."; + + return false; +} + +} // namespace + +StarboardRenderer::StarboardRenderer( + const scoped_refptr& task_runner) + : task_runner_(task_runner), weak_this_(weak_factory_.GetWeakPtr()) { + LOG(INFO) << "StarboardRenderer constructed."; +} + +StarboardRenderer::~StarboardRenderer() { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + + LOG(INFO) << "Destroying StarboardRenderer."; + + // Explicitly reset |player_bridge_| before destroying it. + // With suspend/resume support, some functions in this class may have to + // guard under `if (player_bridge_)`, as the `player_bridge_` can be destroyed + // asynchronously on arbitrary threads. + // This is not necessary when suspend/resume isn't supported, but implement + // in this way just in case. + decltype(player_bridge_) player_bridge; + if (player_bridge_) { + base::AutoLock auto_lock(lock_); + player_bridge = std::move(player_bridge_); + } + player_bridge.reset(); +} + +void StarboardRenderer::Initialize(MediaResource* media_resource, + RendererClient* client, + PipelineStatusCallback init_cb) { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + DCHECK(!media_resource_); + DCHECK(!client_); + DCHECK(!audio_stream_); + DCHECK(!video_stream_); + DCHECK(media_resource); + DCHECK(client); + + LOG(INFO) << "Initializing StarboardRenderer."; + + media_resource_ = media_resource; + client_ = client; + + audio_stream_ = media_resource_->GetFirstStream(DemuxerStream::AUDIO); + video_stream_ = media_resource_->GetFirstStream(DemuxerStream::VIDEO); + + if (audio_stream_ == nullptr && video_stream_ == nullptr) { + LOG(INFO) + << "The video has to contain at least an audio track or a video track."; + std::move(init_cb).Run(PipelineStatus( + DEMUXER_ERROR_NO_SUPPORTED_STREAMS, + "The video has to contain an audio track or a video track.")); + return; + } + + /* + base::AutoLock auto_lock(lock_); + + bool is_encrypted = + audio_stream_ && audio_stream_->audio_decoder_config().is_encrypted(); + is_encrypted |= + video_stream_ && video_stream_->video_decoder_config().is_encrypted(); + + bool natural_size_changed = false; + if (video_stream_) { + natural_size_changed = + (video_stream_->video_decoder_config().natural_size().width() != + natural_size_.width() || + video_stream_->video_decoder_config().natural_size().height() != + natural_size_.height()); + natural_size_ = video_stream_->video_decoder_config().natural_size(); + } + if (natural_size_changed) { + content_size_change_cb_.Run(); + } + + DCHECK(!is_encrypted); // TODO: Support encrypted playbacks + */ + + // |init_cb| will be called inside |CreatePlayerBridge()|. + CreatePlayerBridge(std::move(init_cb)); +} + +void StarboardRenderer::Flush(base::OnceClosure flush_cb) { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + DCHECK(!pending_flush_cb_); + + LOG(INFO) << "Flushing StarboardRenderer."; + + // Prepares the |player_bridge_| for Seek(), the |player_bridge_| won't + // request more data from us before Seek() is called. + player_bridge_->PrepareForSeek(); + + // The function will be called during seek, before Demuxer::Seek() is called. + // It's possible that there are in-flight Demuxer::Read() calls posted in + // OnDemuxerStreamRead(), we will wait until they are finished. + if (audio_read_in_progress_ || video_read_in_progress_) { + pending_flush_cb_ = std::move(flush_cb); + } else { + std::move(flush_cb).Run(); + } +} + +void StarboardRenderer::StartPlayingFrom(base::TimeDelta time) { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + + LOG(INFO) << "StarboardRenderer::StartPlayingFrom() called with " << time; + + if (!player_bridge_initialized_) { + DCHECK(!playing_start_from_time_); + playing_start_from_time_ = time; + return; + } + + player_bridge_->Seek(time); +} + +void StarboardRenderer::SetPlaybackRate(double playback_rate) { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + + LOG(INFO) << "StarboardRenderer changes playback rate from " << playback_rate_ + << " to " << playback_rate << '.'; + + if (playback_rate_ == playback_rate) { + return; + } + + playback_rate_ = playback_rate; + + if (player_bridge_) { + player_bridge_->SetPlaybackRate(playback_rate_); + } +} + +void StarboardRenderer::SetVolume(float volume) { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + + LOG(INFO) << "StarboardRenderer changes volume from " << volume_ << " to " + << volume << '.'; + + if (volume_ == volume) { + return; + } + + volume_ = volume; + + if (player_bridge_) { + player_bridge_->SetVolume(volume_); + } +} + +base::TimeDelta StarboardRenderer::GetMediaTime() { + base::AutoLock auto_lock(lock_); + + if (player_bridge_) { + uint32_t video_frames_decoded, video_frames_dropped; + base::TimeDelta media_time; + + player_bridge_->GetInfo(&video_frames_decoded, &video_frames_dropped, + &media_time); + + PipelineStatistics statistics; + + if (video_frames_decoded > last_video_frames_decoded_) { + statistics.video_frames_decoded = + video_frames_decoded - last_video_frames_decoded_; + last_video_frames_decoded_ = video_frames_decoded; + } + + if (video_frames_dropped > last_video_frames_dropped_) { + statistics.video_frames_dropped = + video_frames_dropped - last_video_frames_dropped_; + last_video_frames_dropped_ = video_frames_dropped; + } + + if (statistics.video_frames_decoded > 0 || + statistics.video_frames_dropped > 0) { + // TODO: Double check if we should report more statistics. + // TODO: Report dropped frames for the last time at the end of playback? + task_runner_->PostTask( + FROM_HERE, base::BindOnce(&RendererClient::OnStatisticsUpdate, + base::Unretained(client_), statistics)); + } + return media_time; + } + return base::Microseconds(0); +} + +void StarboardRenderer::CreatePlayerBridge(PipelineStatusCallback init_cb) { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + DCHECK(audio_stream_ || video_stream_); + + AudioDecoderConfig invalid_audio_config; + const AudioDecoderConfig& audio_config = + audio_stream_ ? audio_stream_->audio_decoder_config() + : invalid_audio_config; + VideoDecoderConfig invalid_video_config; + const VideoDecoderConfig& video_config = + video_stream_ ? video_stream_->video_decoder_config() + : invalid_video_config; + + std::string audio_mime_type = ""; + std::string video_mime_type = ""; + + std::string error_message; + + { + base::AutoLock auto_lock(lock_); + DCHECK(!player_bridge_); + + // In the extreme case that CreatePlayerBridge() is called when a + // |player_bridge_| is set, reset the existing player first to reduce the + // number of active players. + player_bridge_.reset(); + + LOG(INFO) << "Creating SbPlayerBridge."; + + player_bridge_.reset(new SbPlayerBridge( + &sbplayer_interface_, task_runner_, + SbPlayerBridge::GetDecodeTargetGraphicsContextProviderFunc(), + audio_config, audio_mime_type, video_config, video_mime_type, + kSbWindowInvalid, kSbDrmSystemInvalid, this, nullptr, false, + kSbPlayerOutputModeInvalid, "", 1024 * 1024 * 8, + "pipeline_identifier")); + if (player_bridge_->IsValid()) { + // TODO(b/267678497): When `player_bridge_->GetAudioConfigurations()` + // returns no audio configurations, update the write durations again + // before the SbPlayer reaches `kSbPlayerStatePresenting`. + audio_write_duration_ = + HasRemoteAudioOutputs(player_bridge_->GetAudioConfigurations()) + ? audio_write_duration_remote_ + : audio_write_duration_local_; + LOG(INFO) << "SbPlayerBridge created, with audio write duration at " + << audio_write_duration_; + } else { + error_message = player_bridge_->GetPlayerCreationErrorMessage(); + player_bridge_.reset(); + LOG(INFO) << "Failed to create a valid SbPlayerBridge."; + } + } + + if (player_bridge_ && player_bridge_->IsValid()) { + if (audio_stream_) { + UpdateDecoderConfig(audio_stream_); + } + if (video_stream_) { + UpdateDecoderConfig(video_stream_); + } + + std::move(init_cb).Run(PipelineStatus(PIPELINE_OK)); + return; + } + + LOG(INFO) << "SbPlayerPipeline::CreatePlayerBridge() failed to create a" + " valid SbPlayerBridge - \'" + << error_message << "\'"; + + std::move(init_cb).Run(PipelineStatus( + DECODER_ERROR_NOT_SUPPORTED, + "SbPlayerPipeline::CreatePlayerBridge() failed to create a valid" + " SbPlayerBridge - \'" + + error_message + "\'")); +} + +void StarboardRenderer::UpdateDecoderConfig(DemuxerStream* stream) { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + + if (!player_bridge_) { + return; + } + + if (stream->type() == DemuxerStream::AUDIO) { + const AudioDecoderConfig& decoder_config = stream->audio_decoder_config(); + player_bridge_->UpdateAudioConfig(decoder_config, ""); + } else { + DCHECK_EQ(stream->type(), DemuxerStream::VIDEO); + const VideoDecoderConfig& decoder_config = stream->video_decoder_config(); + + /*base::AutoLock auto_lock(lock_); + + bool natural_size_changed = + (decoder_config.natural_size().width() != natural_size_.width() || + decoder_config.natural_size().height() != natural_size_.height()); + + natural_size_ = decoder_config.natural_size(); + + player_bridge_->UpdateVideoConfig(decoder_config, ""); + + if (natural_size_changed) { + content_size_change_cb_.Run(); + }*/ + } +} + +void StarboardRenderer::OnDemuxerStreamRead( + DemuxerStream* stream, + DemuxerStream::Status status, + DemuxerStream::DecoderBufferVector buffers) { + if (!task_runner_->RunsTasksInCurrentSequence()) { + task_runner_->PostTask( + FROM_HERE, base::BindOnce(&StarboardRenderer::OnDemuxerStreamRead, + weak_this_, stream, status, buffers)); + return; + } + + if (pending_flush_cb_) { + if (stream == audio_stream_) { + audio_read_in_progress_ = false; + } + if (stream == video_stream_) { + video_read_in_progress_ = false; + } + if (!audio_read_in_progress_ && !video_read_in_progress_) { + auto flush_cb = std::move(pending_flush_cb_).value(); + std::move(flush_cb).Run(); + } + return; + } + + // if (stopped_) { + // return; + // } + + DCHECK(player_bridge_); + + if (status == DemuxerStream::kOk) { + if (stream == audio_stream_) { + audio_read_in_progress_ = false; + player_bridge_->WriteBuffers(DemuxerStream::AUDIO, buffers); + } else { + video_read_in_progress_ = false; + player_bridge_->WriteBuffers(DemuxerStream::VIDEO, buffers); + } + } else if (status == DemuxerStream::kAborted) { + } else if (status == DemuxerStream::kConfigChanged) { + if (stream == audio_stream_) { + client_->OnAudioConfigChange(stream->audio_decoder_config()); + } else { + DCHECK_EQ(stream, video_stream_); + client_->OnVideoConfigChange(stream->video_decoder_config()); + client_->OnVideoNaturalSizeChange( + stream->video_decoder_config().visible_rect().size()); + } + UpdateDecoderConfig(stream); + stream->Read(1, base::BindOnce(&StarboardRenderer::OnDemuxerStreamRead, + weak_this_, stream)); + } else if (status == DemuxerStream::kError) { + client_->OnError(PIPELINE_ERROR_READ); + } +} + +void StarboardRenderer::OnNeedData(DemuxerStream::Type type, + int max_number_of_buffers_to_write) { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + + // In case if Stop() has been called. + if (!player_bridge_) { + return; + } + + int max_buffers = max_audio_samples_per_write_ > 1 + ? std::min(max_number_of_buffers_to_write, + max_audio_samples_per_write_) + : 1; + + if (type == DemuxerStream::AUDIO) { + if (!audio_stream_) { + LOG(WARNING) + << "Calling OnNeedData() for audio data during audioless playback"; + return; + } + + if (audio_read_in_progress_) { + return; + } +#if 0 + // If we haven't checked the media time recently, update it now. + if (Time::Now() - last_time_media_time_retrieved_ > + kMediaTimeCheckInterval) { + GetMediaTime(); + } + + // Delay reading audio more than |audio_write_duration_| ahead of playback + // after the player has received enough audio for preroll, taking into + // account that our estimate of playback time might be behind by + // |kMediaTimeCheckInterval|. + base::TimeDelta time_ahead_of_playback_for_preroll = + timestamp_of_last_written_audio_ - seek_time_; + auto adjusted_write_duration_for_preroll = + AdjustWriteDurationForPlaybackRate(audio_write_duration_for_preroll_, + playback_rate_); + // Note when Cobalt uses multiple samples per write, GetDefaultMaxBuffers() + // returns the exact number of samples computed by + // |time_ahead_of_playback_for_preroll| and + // |adjusted_write_duration_for_preroll|. The guard number + // kPrerollGuardAudioBuffer is used to ensure Cobalt can do one initial + // write for audio preroll, as preroll condition requires that + // |time_ahead_of_playback_for_preroll| > + // |adjusted_write_duration_for_preroll|. + int estimated_max_buffers = max_buffers; + if (!is_video_eos_written_ && time_ahead_of_playback_for_preroll > + adjusted_write_duration_for_preroll) { + // The estimated time ahead of playback may be negative if no audio has + // been written. + TimeDelta time_ahead_of_playback = + timestamp_of_last_written_audio_ - last_media_time_; + auto adjusted_write_duration = AdjustWriteDurationForPlaybackRate( + audio_write_duration_, playback_rate_); + if (time_ahead_of_playback > + (adjusted_write_duration + kMediaTimeCheckInterval)) { + task_runner_->PostDelayedTask( + FROM_HERE, + base::Bind(&SbPlayerPipeline::DelayedNeedData, this, max_buffers), + kMediaTimeCheckInterval); + audio_read_delayed_ = true; + return; + } + if (max_audio_samples_per_write_ > 1 && + !time_ahead_of_playback.is_negative()) { + estimated_max_buffers = GetEstimatedMaxBuffers(adjusted_write_duration, + time_ahead_of_playback, + false /* is_preroll */); + } + } else if (max_audio_samples_per_write_ > 1) { + if (!time_ahead_of_playback_for_preroll.is_negative()) { + estimated_max_buffers = GetEstimatedMaxBuffers( + adjusted_write_duration_for_preroll, + time_ahead_of_playback_for_preroll, true /* is_preroll */); + last_estimated_max_buffers_for_preroll_ = std::max( + estimated_max_buffers, last_estimated_max_buffers_for_preroll_); + } else { + estimated_max_buffers = last_estimated_max_buffers_for_preroll_; + } + } + // When Cobalt uses multiple samples per write, this ensures that + // |max_buffers| is at most |max_number_of_buffers_to_write|. + // |max_buffers| is in the range of [1, |max_number_of_buffers_to_write|], + // where the lower bound 1 is guarded by GetEstimatedMaxBuffers(). + max_buffers = std::min(max_buffers, estimated_max_buffers); + + audio_read_delayed_ = false; +#endif // 0 + audio_read_in_progress_ = true; + } else { + DCHECK_EQ(type, DemuxerStream::VIDEO); + + if (video_read_in_progress_) { + return; + } + + video_read_in_progress_ = true; + } + auto stream = (type == DemuxerStream::AUDIO ? audio_stream_ : video_stream_); + DCHECK(stream); + + stream->Read(max_buffers, + base::BindOnce(&StarboardRenderer::OnDemuxerStreamRead, + weak_this_, stream)); +} + +void StarboardRenderer::OnPlayerStatus(SbPlayerState state) { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + + // In case if Stop() has been called. + if (!player_bridge_) { + if (state == kSbPlayerStateDestroyed) { + LOG(INFO) << "StarboardRenderer::OnPlayerStatus() called with " + "kSbPlayerStateDestroyed."; + } + return; + } + + switch (state) { + case kSbPlayerStateInitialized: + LOG(INFO) << "StarboardRenderer::OnPlayerStatus() called with " + "kSbPlayerStateInitialized."; + DCHECK(!player_bridge_initialized_); + player_bridge_initialized_ = true; + + if (playing_start_from_time_) { + StartPlayingFrom(std::move(playing_start_from_time_).value()); + } + break; + case kSbPlayerStatePrerolling: + LOG(INFO) << "StarboardRenderer::OnPlayerStatus() called with " + "kSbPlayerStatePrerolling."; + player_bridge_->SetPlaybackRate(playback_rate_); + player_bridge_->SetVolume(volume_); + break; + case kSbPlayerStatePresenting: + client_->OnBufferingStateChange(BUFFERING_HAVE_ENOUGH, + BUFFERING_CHANGE_REASON_UNKNOWN); + // buffering_state_cb_.Run(kPrerollCompleted); + // if (!seek_cb_.is_null()) { + // CallSeekCB(::media::PIPELINE_OK, ""); + // } + audio_write_duration_ = + HasRemoteAudioOutputs(player_bridge_->GetAudioConfigurations()) + ? audio_write_duration_remote_ + : audio_write_duration_local_; + LOG(INFO) << "SbPlayerBridge reaches kSbPlayerStatePresenting, with audio" + << " write duration at " << audio_write_duration_; + break; + case kSbPlayerStateEndOfStream: + LOG(INFO) << "StarboardRenderer::OnPlayerStatus() called with " + "kSbPlayerStateEndOfStream."; + client_->OnEnded(); + break; + case kSbPlayerStateDestroyed: + LOG(INFO) << "StarboardRenderer::OnPlayerStatus() called with " + "kSbPlayerStateDestroyed."; + break; + } +} + +void StarboardRenderer::OnPlayerError(SbPlayerError error, + const std::string& message) { + NOTIMPLEMENTED(); +} + +} // namespace media diff --git a/media/starboard/starboard_renderer.h b/media/starboard/starboard_renderer.h new file mode 100644 index 000000000000..6d03b4856604 --- /dev/null +++ b/media/starboard/starboard_renderer.h @@ -0,0 +1,136 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIA_STARBOARD_STARBOARD_RENDERER_H_ +#define MEDIA_STARBOARD_STARBOARD_RENDERER_H_ + +#include + +#include "base/memory/raw_ptr.h" +#include "base/memory/scoped_refptr.h" +#include "base/memory/weak_ptr.h" +#include "base/synchronization/lock.h" +#include "base/task/sequenced_task_runner.h" +#include "base/time/time.h" +#include "media/base/cdm_context.h" +#include "media/base/decoder_buffer.h" +#include "media/base/demuxer_stream.h" +#include "media/base/media_resource.h" +#include "media/base/pipeline_status.h" +#include "media/base/renderer.h" +#include "media/base/renderer_client.h" +#include "media/starboard/sbplayer_bridge.h" +#include "third_party/abseil-cpp/absl/types/optional.h" + +namespace media { + +class MEDIA_EXPORT StarboardRenderer final : public Renderer, + private SbPlayerBridge::Host { + public: + explicit StarboardRenderer( + const scoped_refptr& task_runner); + + ~StarboardRenderer() final; + + // Renderer implementation. + void Initialize(MediaResource* media_resource, + RendererClient* client, + PipelineStatusCallback init_cb) final; + void SetCdm(CdmContext* cdm_context, CdmAttachedCB cdm_attached_cb) final { + NOTIMPLEMENTED(); + } + void SetLatencyHint(absl::optional latency_hint) final { + NOTIMPLEMENTED(); + } + void SetPreservesPitch(bool preserves_pitch) final { NOTIMPLEMENTED(); } + void SetWasPlayedWithUserActivation( + bool was_played_with_user_activation) final { + NOTIMPLEMENTED(); + } + void Flush(base::OnceClosure flush_cb) final; + void StartPlayingFrom(base::TimeDelta time) final; + void SetPlaybackRate(double playback_rate) final; + void SetVolume(float volume) final; + base::TimeDelta GetMediaTime() final; + void OnSelectedVideoTracksChanged( + const std::vector& enabled_tracks, + base::OnceClosure change_completed_cb) final { + NOTIMPLEMENTED(); + } + void OnEnabledAudioTracksChanged( + const std::vector& enabled_tracks, + base::OnceClosure change_completed_cb) final { + NOTIMPLEMENTED(); + } + RendererType GetRendererType() final { + // Reuse `kRendererImpl` to avoid introducing a new renderer type. + return RendererType::kRendererImpl; + } + + private: + void CreatePlayerBridge(PipelineStatusCallback init_cb); + void UpdateDecoderConfig(DemuxerStream* stream); + void OnDemuxerStreamRead(DemuxerStream* stream, + DemuxerStream::Status status, + DemuxerStream::DecoderBufferVector buffers); + + void OnNeedData(DemuxerStream::Type type, + int max_number_of_buffers_to_write) final; + void OnPlayerStatus(SbPlayerState state) final; + void OnPlayerError(SbPlayerError error, const std::string& message) final; + + scoped_refptr task_runner_; + + raw_ptr media_resource_ = nullptr; + raw_ptr audio_stream_ = nullptr; + raw_ptr video_stream_ = nullptr; + raw_ptr client_ = nullptr; + + DefaultSbPlayerInterface sbplayer_interface_; + const base::TimeDelta audio_write_duration_local_ = base::Milliseconds(500); + const base::TimeDelta audio_write_duration_remote_ = base::Seconds(10); + const int max_audio_samples_per_write_ = 1; + + base::Lock lock_; + std::unique_ptr player_bridge_; + + bool player_bridge_initialized_ = false; + std::optional playing_start_from_time_; + + std::optional pending_flush_cb_; + + base::TimeDelta audio_write_duration_ = audio_write_duration_local_; + + bool audio_read_in_progress_ = false; + bool video_read_in_progress_ = false; + + // TODO: Consider calling `void OnWaiting(WaitingReason reason)` on `client_`. + // TODO: Shall we call `void OnVideoFrameRateChange(absl::optional fps)` + // on `client_`? + double playback_rate_ = 0.; + float volume_ = 1.0f; + + uint32_t last_video_frames_decoded_ = 0; + uint32_t last_video_frames_dropped_ = 0; + + base::WeakPtrFactory weak_factory_{this}; + base::WeakPtr weak_this_; + + StarboardRenderer(const StarboardRenderer&) = delete; + StarboardRenderer& operator=(const StarboardRenderer&) = delete; +}; + +} // namespace media + +#endif // MEDIA_STARBOARD_STARBOARD_RENDERER_H_ diff --git a/media/starboard/starboard_utils.cc b/media/starboard/starboard_utils.cc index 25871101dedc..c529c57ea8b0 100644 --- a/media/starboard/starboard_utils.cc +++ b/media/starboard/starboard_utils.cc @@ -16,7 +16,7 @@ #error "This file only works with Cobalt/Starboard." #endif // !defined(STARBOARD) -#include "media/base/starboard_utils.h" +#include "media/starboard/starboard_utils.h" #include @@ -74,15 +74,15 @@ SbMediaAudioCodec MediaAudioCodecToSbMediaAudioCodec(AudioCodec codec) { return kSbMediaAudioCodecFlac; case AudioCodec::kPCM: return kSbMediaAudioCodecPcm; +#if COBALT_MEDIA_ENABLE_IAMF_SUPPORT case AudioCodec::kIAMF: return kSbMediaAudioCodecIamf; +#endif // COBALT_MEDIA_ENABLE_IAMF_SUPPORT default: // Cobalt only supports a subset of audio codecs defined by Chromium. - DLOG(ERROR) << "Unsupported audio codec " << GetCodecName(codec); + LOG(ERROR) << "Unsupported audio codec " << GetCodecName(codec); return kSbMediaAudioCodecNone; } - NOTREACHED(); - return kSbMediaAudioCodecNone; } SbMediaVideoCodec MediaVideoCodecToSbMediaVideoCodec(VideoCodec codec) { @@ -105,11 +105,9 @@ SbMediaVideoCodec MediaVideoCodecToSbMediaVideoCodec(VideoCodec codec) { return kSbMediaVideoCodecAv1; default: // Cobalt only supports a subset of video codecs defined by Chromium. - DLOG(ERROR) << "Unsupported video codec " << GetCodecName(codec); + LOG(ERROR) << "Unsupported video codec " << GetCodecName(codec); return kSbMediaVideoCodecNone; } - NOTREACHED(); - return kSbMediaVideoCodecNone; } SbMediaAudioStreamInfo MediaAudioConfigToSbMediaAudioStreamInfo( @@ -126,7 +124,8 @@ SbMediaAudioStreamInfo MediaAudioConfigToSbMediaAudioStreamInfo( ChannelLayoutToChannelCount(audio_decoder_config.channel_layout()); audio_stream_info.samples_per_second = audio_decoder_config.samples_per_second(); - audio_stream_info.bits_per_sample = audio_decoder_config.bits_per_channel(); + audio_stream_info.bits_per_sample = + audio_decoder_config.bytes_per_channel() * 8; const auto& extra_data = audio_stream_info.codec == kSbMediaAudioCodecAac ? audio_decoder_config.aac_extra_data() @@ -201,8 +200,8 @@ void FillDrmSampleInfo(const scoped_refptr& buffer, drm_info->subsample_count = config->subsamples().size(); if (drm_info->subsample_count > 0) { - COMPILE_ASSERT(sizeof(SbDrmSubSampleMapping) == sizeof(SubsampleEntry), - SubSampleEntrySizesMatch); + static_assert(sizeof(SbDrmSubSampleMapping) == sizeof(SubsampleEntry), + "SubSampleEntrySizesMatch"); drm_info->subsample_mapping = reinterpret_cast( &config->subsamples()[0]); @@ -223,8 +222,10 @@ void FillDrmSampleInfo(const scoped_refptr& buffer, // Ensure that the enums in starboard/media.h match enums in // VideoColorSpace and gfx::ColorSpace. -#define ENUM_EQ(a, b) \ - COMPILE_ASSERT(static_cast(a) == static_cast(b), mismatching_enums) +#define ENUM_EQ(a, b) \ + static_assert(static_cast(a) == static_cast(b), \ + "mismatching_" \ + "enums") // Ensure PrimaryId enums convert correctly. ENUM_EQ(kSbMediaPrimaryIdReserved0, VideoColorSpace::PrimaryID::INVALID); @@ -366,6 +367,7 @@ SbMediaColorMetadata MediaToSbMediaColorMetadata( } int GetSbMediaVideoBufferBudget(const VideoDecoderConfig* video_config, const std::string& mime_type) { +#if COBALT_MEDIA_ENABLE_DECODE_BUFFER_BUDGET if (!video_config) { return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget( kSbMediaVideoCodecH264, 1920, 1080, 8); @@ -377,6 +379,9 @@ int GetSbMediaVideoBufferBudget(const VideoDecoderConfig* video_config, auto codec = MediaVideoCodecToSbMediaVideoCodec(video_config->codec()); return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget( codec, width, height, bits_per_pixel); +#else // COBALT_MEDIA_ENABLE_DECODE_BUFFER_BUDGET + return 100 * 1024 * 1024; +#endif // COBALT_MEDIA_ENABLE_DECODE_BUFFER_BUDGET } std::string ExtractCodecs(const std::string& mime_type) { diff --git a/media/starboard/starboard_utils.h b/media/starboard/starboard_utils.h index 87119183ce0c..8f2a174efc03 100644 --- a/media/starboard/starboard_utils.h +++ b/media/starboard/starboard_utils.h @@ -12,12 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef MEDIA_BASE_STARBOARD_UTILS_H_ -#define MEDIA_BASE_STARBOARD_UTILS_H_ - -#if !defined(STARBOARD) -#error "This file only works with Cobalt/Starboard." -#endif // !defined(STARBOARD) +#ifndef MEDIA_STARBOARD_STARBOARD_UTILS_H_ +#define MEDIA_STARBOARD_STARBOARD_UTILS_H_ #include "media/base/audio_codecs.h" #include "media/base/audio_decoder_config.h" @@ -65,4 +61,4 @@ std::string ExtractCodecs(const std::string& mime_type); } // namespace media -#endif // MEDIA_BASE_STARBOARD_UTILS_H_ +#endif // MEDIA_STARBOARD_STARBOARD_UTILS_H_ diff --git a/media/starboard/starboard_utils_test.cc b/media/starboard/starboard_utils_test.cc index a2feb6d2e101..88368892a5cc 100644 --- a/media/starboard/starboard_utils_test.cc +++ b/media/starboard/starboard_utils_test.cc @@ -16,7 +16,7 @@ #error "This file only works with Cobalt/Starboard." #endif // !defined(STARBOARD) -#include "media/base/starboard_utils.h" +#include "media/starboard/starboard_utils.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/third_party/blink/renderer/modules/mediasource/BUILD.gn b/third_party/blink/renderer/modules/mediasource/BUILD.gn index 513d0e9d92b7..1eb0a04977eb 100644 --- a/third_party/blink/renderer/modules/mediasource/BUILD.gn +++ b/third_party/blink/renderer/modules/mediasource/BUILD.gn @@ -52,4 +52,7 @@ blink_modules_sources("mediasource") { # Ensure the generated webcodecs v8 config and chunk bindings are available. "//third_party/blink/renderer/modules/webcodecs:webcodecs", ] + if (is_cobalt) { + deps += [ "//starboard/build:starboard_buildflags" ] + } } diff --git a/third_party/blink/renderer/modules/mediasource/media_source.cc b/third_party/blink/renderer/modules/mediasource/media_source.cc index bffaac9e4631..145945d42c08 100644 --- a/third_party/blink/renderer/modules/mediasource/media_source.cc +++ b/third_party/blink/renderer/modules/mediasource/media_source.cc @@ -20,6 +20,8 @@ #include "media/base/supported_types.h" #include "media/base/video_decoder_config.h" #include "media/media_buildflags.h" +// For BUILDFLAG(USE_STARBOARD_MEDIA) +#include "starboard/build/starboard_buildflags.h" #include "third_party/blink/public/common/privacy_budget/identifiability_metric_builder.h" #include "third_party/blink/public/common/privacy_budget/identifiability_study_settings.h" #include "third_party/blink/public/common/privacy_budget/identifiable_surface.h" @@ -492,6 +494,10 @@ bool MediaSource::isTypeSupported(ExecutionContext* context, bool result = IsTypeSupportedInternal( context, type, true /* Require fully specified mime and codecs */); DVLOG(2) << __func__ << "(" << type << ") -> " << (result ? "true" : "false"); +#if BUILDFLAG(USE_STARBOARD_MEDIA) + LOG(INFO) << __func__ << "(" << type << ") -> " + << (result ? "true" : "false"); +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) return result; } @@ -499,6 +505,40 @@ bool MediaSource::isTypeSupported(ExecutionContext* context, bool MediaSource::IsTypeSupportedInternal(ExecutionContext* context, const String& type, bool enforce_codec_specificity) { +#if BUILDFLAG(USE_STARBOARD_MEDIA) + // TODO(b/322021829): This is a temporary workaround to claim 4K playback + // support. It should be replaced by a proper implementation of + // MediaSource.isTypeSupported(). + if (type.Find("99") != kNotFound || type.Find("catavision") != kNotFound || + type.Find("invalidformat") != kNotFound || + type.Find("bitrate=2000000000") != kNotFound || + type.Find("decode-to-texture=nope") != kNotFound || + type.Find("decode-to-texture=true") != kNotFound) { + return false; + } + + // Reject 8k + if (type.Find("7680") != kNotFound) { + return false; + } + + /* + // Reject 4k + if (type.Find("3840") != kNotFound) { + return false; + } + + // Reject 2k + if (type.Find("2560") != kNotFound) { + return false; + }*/ + + // Reject av1, as it's flaky on my test device + // if (type.Find("av1") != kNotFound || type.Find("av01") != kNotFound) { + // return false; + // } +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) + // Even after ExecutionContext teardown notification, bindings may still call // code-behinds for a short while. If |context| is null, this is likely // happening. To prevent possible null deref of |context| in this path, claim diff --git a/third_party/blink/renderer/platform/media/BUILD.gn b/third_party/blink/renderer/platform/media/BUILD.gn index f679e9348ee0..d8e88fcd6b8d 100644 --- a/third_party/blink/renderer/platform/media/BUILD.gn +++ b/third_party/blink/renderer/platform/media/BUILD.gn @@ -104,6 +104,9 @@ component("media") { "//media/remoting:remoting_constants", "//third_party/blink/public/strings:strings_grit", ] + if (is_cobalt) { + deps += [ "//starboard/build:starboard_buildflags" ] + } } if (enable_hls_demuxer) { diff --git a/third_party/blink/renderer/platform/media/web_media_player_impl.cc b/third_party/blink/renderer/platform/media/web_media_player_impl.cc index d0b3f2bf6f23..d87712349771 100644 --- a/third_party/blink/renderer/platform/media/web_media_player_impl.cc +++ b/third_party/blink/renderer/platform/media/web_media_player_impl.cc @@ -65,6 +65,8 @@ #include "mojo/public/cpp/bindings/pending_remote.h" #include "net/base/data_url.h" #include "services/device/public/mojom/battery_monitor.mojom-blink.h" +// For BUILDFLAG(USE_STARBOARD_MEDIA) +#include "starboard/build/starboard_buildflags.h" #include "third_party/blink/public/common/media/display_type.h" #include "third_party/blink/public/common/media/watch_time_reporter.h" #include "third_party/blink/public/common/thread_safe_browser_interface_broker_proxy.h" @@ -98,6 +100,10 @@ #include "third_party/blink/renderer/platform/media/web_media_source_impl.h" #include "ui/gfx/geometry/size.h" +#if BUILDFLAG(USE_STARBOARD_MEDIA) +#include "media/starboard/starboard_renderer.h" +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) + #if BUILDFLAG(ENABLE_HLS_DEMUXER) #include "third_party/blink/renderer/platform/media/hls_data_source_provider_impl.h" #endif // BUILDFLAG(ENABLE_HLS_DEMUXER) @@ -2796,6 +2802,15 @@ std::unique_ptr WebMediaPlayerImpl::CreateRenderer( &WebMediaPlayerImpl::OnOverlayInfoRequested, weak_this_)); #endif +#if BUILDFLAG(USE_STARBOARD_MEDIA) + // TODO(): Properly branching on the StarboardRenderer. + // `media_task_runner_` is always true, use an if statement to avoid + // potential build warning on unreachable code. + if (media_task_runner_) { + return std::make_unique(media_task_runner_); + } +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) + if (renderer_type) { DVLOG(1) << __func__ << ": renderer_type=" << static_cast(renderer_type.value());