From 3e56903d008b472a63471ec753009b93c51da377 Mon Sep 17 00:00:00 2001 From: Xiaoming Shi Date: Mon, 21 Oct 2024 10:51:12 -0700 Subject: [PATCH] [media] Implement SbPlayer based video playback The implementation was based on SbPlayerPipeline, with many features disabled (e.g. suspend/resume, UMA metrics) or unimplemented. The former are tracked by feature macros in //media/starboard/BUILD.gn linked with tracking bugs, the latters are marked with TODOs with tracking bugs. It also 1. Removed support of old Starboard versions (e.g. the EnhancedAudio starboard extension). 2. Worked around for incomplete type in SbPlayerBridge::CallbackHelper (see b/375665361). b/328305706 --- media/BUILD.gn | 4 + media/media_options.gni | 4 + media/starboard/BUILD.gn | 96 + media/starboard/sbplayer_bridge.cc | 234 ++- media/starboard/sbplayer_bridge.h | 78 +- media/starboard/sbplayer_interface.cc | 36 +- media/starboard/sbplayer_interface.h | 65 +- media/starboard/sbplayer_set_bounds_helper.cc | 6 +- media/starboard/sbplayer_set_bounds_helper.h | 17 +- media/starboard/starboard_renderer.cc | 1560 ++++------------- media/starboard/starboard_renderer.h | 435 ++--- media/starboard/starboard_utils.cc | 33 +- media/starboard/starboard_utils.h | 10 +- media/starboard/starboard_utils_test.cc | 2 +- starboard/android/shared/main.cc | 3 + .../renderer/modules/mediasource/BUILD.gn | 3 + .../modules/mediasource/media_source.cc | 41 + .../blink/renderer/platform/media/BUILD.gn | 3 + .../platform/media/web_media_player_impl.cc | 24 + 19 files changed, 873 insertions(+), 1781 deletions(-) create mode 100644 media/starboard/BUILD.gn diff --git a/media/BUILD.gn b/media/BUILD.gn index ba14ef6f6b07..0f5ea42427f8 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -202,6 +202,10 @@ test("media_unittests") { "//media/webrtc:unit_tests", ] + if (is_cobalt) { + deps += ["//media/starboard:unit_tests"] + } + data = [ "test/data/", "formats/mp4/h264_annex_b_fuzz_corpus/", diff --git a/media/media_options.gni b/media/media_options.gni index cdc3e9375339..694f26f5e7fe 100644 --- a/media/media_options.gni +++ b/media/media_options.gni @@ -345,6 +345,10 @@ media_subcomponent_deps = [ "//media/video", ] +if (is_cobalt) { + media_subcomponent_deps += [ "//media/starboard" ] +} + if (is_fuchsia) { media_subcomponent_deps += [ "//media/fuchsia/common" ] } diff --git a/media/starboard/BUILD.gn b/media/starboard/BUILD.gn new file mode 100644 index 000000000000..c3b622d38901 --- /dev/null +++ b/media/starboard/BUILD.gn @@ -0,0 +1,96 @@ +# Copyright 2024 The Cobalt Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source_set("starboard") { + visibility = [ "//media" ] + + defines = [ + "STARBOARD", + + # TODO(b/326652276): Support audio write ahead + "COBALT_MEDIA_ENABLE_AUDIO_DURATION=0", + + # TODO(b/326508279): Revisit background mode + "COBALT_MEDIA_ENABLE_BACKGROUND_MODE=0", + + # TODO(b/375069564): Revisit CValStats + "COBALT_MEDIA_ENABLE_CVAL=0", + + # TODO(b/374822966): Revisit DecoderBuffer budget + "COBALT_MEDIA_ENABLE_DECODE_BUFFER_BUDGET=0", + + # TODO(b/375070492): Revisit decode-to-texture + "COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER=0", + + # TODO(b/328305808): Enable encrypted playbacks + "COBALT_MEDIA_ENABLE_ENCRYPTED_PLAYBACKS=0", + + # TODO(b/375218518): Revisit FormatSupportQueryMetrics + "COBALT_MEDIA_ENABLE_FORMAT_SUPPORT_QUERY_METRICS=0", + + # TODO(b/375232937): Enable IAMF + "COBALT_MEDIA_ENABLE_IAMF_SUPPORT=0", + + # TODO(b/326654546): Revisit max video input size + "COBALT_MEDIA_ENABLE_PLAYER_SET_MAX_VIDEO_INPUT_SIZE=0", + + # TODO(b/375059645): Revisit side data (aka hdr10+) support + "COBALT_MEDIA_ENABLE_SIDE_DATA=0", + + # TODO(b/375234428): Revisit startup latency tracking + "COBALT_MEDIA_ENABLE_STARTUP_LATENCY_TRACKING=0", + + # TODO(b/326497953): Revisit suspend/resume support + "COBALT_MEDIA_ENABLE_SUSPEND_RESUME=0", + + # TODO(b/375241265): Re-enable UMA metrics + "COBALT_MEDIA_ENABLE_UMA_METRICS=0", + ] + + sources = [ + "sbplayer_bridge.cc", + "sbplayer_bridge.h", + "sbplayer_interface.cc", + "sbplayer_interface.h", + "sbplayer_set_bounds_helper.cc", + "sbplayer_set_bounds_helper.h", + "starboard_renderer.cc", + "starboard_renderer.h", + "starboard_utils.cc", + "starboard_utils.h", + ] + + deps = [ + "//base", + "//media:media_buildflags", + "//media/base", + "//starboard($starboard_toolchain)", + ] + + configs += [ "//media:subcomponent_config" ] +} + +source_set("unit_tests") { + testonly = true + sources = [ "starboard_utils_test.cc" ] + configs += [ "//media:media_config" ] + deps = [ + "//base", + "//base/test:test_support", + "//media:test_support", + "//starboard($starboard_toolchain)", + "//testing/gmock", + "//testing/gtest", + ] +} diff --git a/media/starboard/sbplayer_bridge.cc b/media/starboard/sbplayer_bridge.cc index 44173baab20a..eac083a91651 100644 --- a/media/starboard/sbplayer_bridge.cc +++ b/media/starboard/sbplayer_bridge.cc @@ -12,30 +12,35 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "cobalt/media/base/sbplayer_bridge.h" +#include "media/starboard/sbplayer_bridge.h" #include #include #include #include -#include "base/bind.h" #include "base/compiler_specific.h" +#include "base/functional/bind.h" #include "base/location.h" #include "base/logging.h" #include "base/metrics/histogram_functions.h" #include "base/trace_event/trace_event.h" +#if COBALT_MEDIA_ENABLE_STARTUP_LATENCY_TRACKING #include "cobalt/base/statistics.h" +#endif // COBALT_MEDIA_ENABLE_STARTUP_LATENCY_TRACKING +#if COBALT_MEDIA_ENABLE_FORMAT_SUPPORT_QUERY_METRICS #include "cobalt/media/base/format_support_query_metrics.h" -#include "media/base/starboard_utils.h" +#endif // COBALT_MEDIA_ENABLE_FORMAT_SUPPORT_QUERY_METRICS +#include "media/starboard/starboard_utils.h" #include "starboard/common/media.h" #include "starboard/common/once.h" #include "starboard/common/player.h" #include "starboard/common/string.h" #include "starboard/configuration.h" +#if COBALT_MEDIA_ENABLE_PLAYER_SET_MAX_VIDEO_INPUT_SIZE #include "starboard/extension/player_set_max_video_input_size.h" +#endif // COBALT_MEDIA_ENABLE_PLAYER_SET_MAX_VIDEO_INPUT_SIZE -namespace cobalt { namespace media { namespace { @@ -45,28 +50,14 @@ using base::TimeDelta; using starboard::FormatString; using starboard::GetPlayerOutputModeName; +#if COBALT_MEDIA_ENABLE_STARTUP_LATENCY_TRACKING class StatisticsWrapper { public: static StatisticsWrapper* GetInstance(); base::Statistics startup_latency{ "Media.PlaybackStartupLatency"}; }; - -void SetStreamInfo( - const SbMediaAudioStreamInfo& stream_info, - CobaltExtensionEnhancedAudioMediaAudioSampleInfo* sample_info) { - DCHECK(sample_info); - - sample_info->stream_info.codec = stream_info.codec; - sample_info->stream_info.mime = stream_info.mime; - sample_info->stream_info.number_of_channels = stream_info.number_of_channels; - sample_info->stream_info.samples_per_second = stream_info.samples_per_second; - sample_info->stream_info.bits_per_sample = stream_info.bits_per_sample; - sample_info->stream_info.audio_specific_config_size = - stream_info.audio_specific_config_size; - sample_info->stream_info.audio_specific_config = - stream_info.audio_specific_config; -} +#endif // COBALT_MEDIA_ENABLE_STARTUP_LATENCY_TRACKING void SetStreamInfo(const SbMediaAudioStreamInfo& stream_info, SbMediaAudioSampleInfo* sample_info) { @@ -75,36 +66,12 @@ void SetStreamInfo(const SbMediaAudioStreamInfo& stream_info, sample_info->stream_info = stream_info; } -void SetStreamInfo( - const SbMediaVideoStreamInfo& stream_info, - CobaltExtensionEnhancedAudioMediaVideoSampleInfo* sample_info) { - DCHECK(sample_info); - - sample_info->stream_info.codec = stream_info.codec; - sample_info->stream_info.mime = stream_info.mime; - sample_info->stream_info.max_video_capabilities = - stream_info.max_video_capabilities; - sample_info->stream_info.frame_width = stream_info.frame_width; - sample_info->stream_info.frame_height = stream_info.frame_height; -} - void SetStreamInfo(const SbMediaVideoStreamInfo& stream_info, SbMediaVideoSampleInfo* sample_info) { DCHECK(sample_info); sample_info->stream_info = stream_info; } -void SetDiscardPadding( - const ::media::DecoderBuffer::DiscardPadding& discard_padding, - CobaltExtensionEnhancedAudioMediaAudioSampleInfo* sample_info) { - DCHECK(sample_info); - - sample_info->discarded_duration_from_front = - discard_padding.first.InMicroseconds(); - sample_info->discarded_duration_from_back = - discard_padding.second.InMicroseconds(); -} - void SetDiscardPadding( const ::media::DecoderBuffer::DiscardPadding& discard_padding, SbMediaAudioSampleInfo* sample_info) { @@ -118,7 +85,9 @@ void SetDiscardPadding( } // namespace +#if COBALT_MEDIA_ENABLE_STARTUP_LATENCY_TRACKING SB_ONCE_INITIALIZE_FUNCTION(StatisticsWrapper, StatisticsWrapper::GetInstance); +#endif // COBALT_MEDIA_ENABLE_STARTUP_LATENCY_TRACKING SbPlayerBridge::CallbackHelper::CallbackHelper(SbPlayerBridge* player_bridge) : player_bridge_(player_bridge) {} @@ -130,7 +99,7 @@ void SbPlayerBridge::CallbackHelper::ClearDecoderBufferCache() { } } -void SbPlayerBridge::CallbackHelper::OnDecoderStatus(SbPlayer player, +void SbPlayerBridge::CallbackHelper::OnDecoderStatus(void* player, SbMediaType type, SbPlayerDecoderState state, int ticket) { @@ -141,21 +110,23 @@ void SbPlayerBridge::CallbackHelper::OnDecoderStatus(SbPlayer player, } } -void SbPlayerBridge::CallbackHelper::OnPlayerStatus(SbPlayer player, +void SbPlayerBridge::CallbackHelper::OnPlayerStatus(void* player, SbPlayerState state, int ticket) { base::AutoLock auto_lock(lock_); if (player_bridge_) { - player_bridge_->OnPlayerStatus(player, state, ticket); + player_bridge_->OnPlayerStatus(static_cast(player), state, + ticket); } } -void SbPlayerBridge::CallbackHelper::OnPlayerError(SbPlayer player, +void SbPlayerBridge::CallbackHelper::OnPlayerError(void* player, SbPlayerError error, const std::string& message) { base::AutoLock auto_lock(lock_); if (player_bridge_) { - player_bridge_->OnPlayerError(player, error, message); + player_bridge_->OnPlayerError(static_cast(player), error, + message); } } @@ -208,10 +179,12 @@ SbPlayerBridge::SbPlayerBridge( CreateUrlPlayer(url_); +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME task_runner->PostTask( - FROM_HERE, - base::Bind(&SbPlayerBridge::CallbackHelper::ClearDecoderBufferCache, - callback_helper_)); + FROM_HERE, base::BindRepeatedly( + &SbPlayerBridge::CallbackHelper::ClearDecoderBufferCache, + callback_helper_)); +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME } #endif // SB_HAS(PLAYER_WITH_URL) @@ -230,16 +203,21 @@ SbPlayerBridge::SbPlayerBridge( SbPlayerSetBoundsHelper* set_bounds_helper, bool allow_resume_after_suspend, SbPlayerOutputMode default_output_mode, +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER DecodeTargetProvider* const decode_target_provider, +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER const std::string& max_video_capabilities, - int max_video_input_size, - std::string pipeline_identifier) + int max_video_input_size +#if COBALT_MEDIA_ENABLE_CVAL + , + std::string pipeline_identifier +#endif // COBALT_MEDIA_ENABLE_CVAL + ) : sbplayer_interface_(interface), task_runner_(task_runner), get_decode_target_graphics_context_provider_func_( get_decode_target_graphics_context_provider_func), - callback_helper_( - new CallbackHelper(ALLOW_THIS_IN_INITIALIZER_LIST(this))), + callback_helper_(new CallbackHelper(this)), window_(window), drm_system_(drm_system), host_(host), @@ -247,21 +225,28 @@ SbPlayerBridge::SbPlayerBridge( allow_resume_after_suspend_(allow_resume_after_suspend), audio_config_(audio_config), video_config_(video_config), +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER decode_target_provider_(decode_target_provider), +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER max_video_capabilities_(max_video_capabilities), - max_video_input_size_(max_video_input_size), - cval_stats_(&interface->cval_stats_), - pipeline_identifier_(pipeline_identifier) + max_video_input_size_(max_video_input_size) +#if COBALT_MEDIA_ENABLE_CVAL + cval_stats_(&interface->cval_stats_), + pipeline_identifier_(pipeline_identifier), +#endif // COBALT_MEDIA_ENABLE_CVAL #if SB_HAS(PLAYER_WITH_URL) - , is_url_based_(false) #endif // SB_HAS(PLAYER_WITH_URL { +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER DCHECK(!get_decode_target_graphics_context_provider_func_.is_null()); +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER DCHECK(audio_config.IsValidConfig() || video_config.IsValidConfig()); DCHECK(host_); DCHECK(set_bounds_helper_); +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER DCHECK(decode_target_provider_); +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER audio_stream_info_.codec = kSbMediaAudioCodecNone; video_stream_info_.codec = kSbMediaVideoCodecNone; @@ -278,12 +263,14 @@ SbPlayerBridge::SbPlayerBridge( CreatePlayer(); +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME if (SbPlayerIsValid(player_)) { task_runner->PostTask( - FROM_HERE, - base::Bind(&SbPlayerBridge::CallbackHelper::ClearDecoderBufferCache, - callback_helper_)); + FROM_HERE, base::BindRepeatedly( + &SbPlayerBridge::CallbackHelper::ClearDecoderBufferCache, + callback_helper_)); } +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME } SbPlayerBridge::~SbPlayerBridge() { @@ -292,14 +279,20 @@ SbPlayerBridge::~SbPlayerBridge() { callback_helper_->ResetPlayer(); set_bounds_helper_->SetPlayerBridge(NULL); +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER decode_target_provider_->SetOutputMode( DecodeTargetProvider::kOutputModeInvalid); decode_target_provider_->ResetGetCurrentSbDecodeTargetFunction(); +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER if (SbPlayerIsValid(player_)) { +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StartTimer(MediaTiming::SbPlayerDestroy, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL sbplayer_interface_->Destroy(player_); +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StopTimer(MediaTiming::SbPlayerDestroy, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL } } @@ -350,6 +343,7 @@ void SbPlayerBridge::WriteBuffers( DCHECK(!is_url_based_); #endif // SB_HAS(PLAYER_WITH_URL) +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME if (allow_resume_after_suspend_) { if (type == DemuxerStream::Type::AUDIO) { decoder_buffer_cache_.AddBuffers(buffers, audio_stream_info_); @@ -362,14 +356,10 @@ void SbPlayerBridge::WriteBuffers( } return; } +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME - if (sbplayer_interface_->IsEnhancedAudioExtensionEnabled()) { - WriteBuffersInternal( - type, buffers, &audio_stream_info_, &video_stream_info_); - } else { - WriteBuffersInternal(type, buffers, &audio_stream_info_, - &video_stream_info_); - } + WriteBuffersInternal(type, buffers, &audio_stream_info_, + &video_stream_info_); } void SbPlayerBridge::SetBounds(int z_index, const gfx::Rect& rect) { @@ -401,7 +391,9 @@ void SbPlayerBridge::PrepareForSeek() { void SbPlayerBridge::Seek(TimeDelta time) { DCHECK(task_runner_->RunsTasksInCurrentSequence()); +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME decoder_buffer_cache_.ClearAll(); +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME seek_pending_ = false; pending_audio_eos_buffer_ = false; @@ -455,8 +447,8 @@ void SbPlayerBridge::SetPlaybackRate(double playback_rate) { sbplayer_interface_->SetPlaybackRate(player_, playback_rate); } -void SbPlayerBridge::GetInfo(uint32* video_frames_decoded, - uint32* video_frames_dropped, +void SbPlayerBridge::GetInfo(uint32_t* video_frames_decoded, + uint32_t* video_frames_dropped, TimeDelta* media_time) { DCHECK(video_frames_decoded || video_frames_dropped || media_time); @@ -604,13 +596,19 @@ void SbPlayerBridge::Suspend() { state_ = kSuspended; +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER decode_target_provider_->SetOutputMode( DecodeTargetProvider::kOutputModeInvalid); decode_target_provider_->ResetGetCurrentSbDecodeTargetFunction(); +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StartTimer(MediaTiming::SbPlayerDestroy, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL sbplayer_interface_->Destroy(player_); +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StopTimer(MediaTiming::SbPlayerDestroy, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL player_ = kSbPlayerInvalid; } @@ -626,7 +624,9 @@ void SbPlayerBridge::Resume(SbWindow window) { return; } +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME decoder_buffer_cache_.StartResuming(); +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME #if SB_HAS(PLAYER_WITH_URL) if (is_url_based_) { @@ -648,7 +648,10 @@ void SbPlayerBridge::Resume(SbWindow window) { } } +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER + namespace { + DecodeTargetProvider::OutputMode ToVideoFrameProviderOutputMode( SbPlayerOutputMode output_mode) { switch (output_mode) { @@ -666,6 +669,8 @@ DecodeTargetProvider::OutputMode ToVideoFrameProviderOutputMode( } // namespace +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER + #if SB_HAS(PLAYER_WITH_URL) // static void SbPlayerBridge::EncryptedMediaInitDataEncounteredCB( @@ -721,12 +726,16 @@ void SbPlayerBridge::CreateUrlPlayer(const std::string& url) { UpdateBounds_Locked(); } #endif // SB_HAS(PLAYER_WITH_URL) + void SbPlayerBridge::CreatePlayer() { - TRACE_EVENT0("cobalt::media", "SbPlayerBridge::CreatePlayer"); DCHECK(task_runner_->RunsTasksInCurrentSequence()); + TRACE_EVENT0("media", "SbPlayerBridge::CreatePlayer"); +#if COBALT_MEDIA_ENABLE_BACKGROUND_MODE bool is_visible = SbWindowIsValid(window_); - bool has_audio = audio_stream_info_.codec != kSbMediaAudioCodecNone; +#else // COBALT_MEDIA_ENABLE_BACKGROUND_MODE + bool is_visible = true; +#endif // COBALT_MEDIA_ENABLE_BACKGROUND_MODE is_creating_player_ = true; @@ -739,9 +748,11 @@ void SbPlayerBridge::CreatePlayer() { return; } +#if COBALT_MEDIA_ENABLE_FORMAT_SUPPORT_QUERY_METRICS if (max_video_capabilities_.empty()) { FormatSupportQueryMetrics::PrintAndResetMetrics(); } +#endif // COBALT_MEDIA_ENABLE_FORMAT_SUPPORT_QUERY_METRICS player_creation_time_ = Time::Now(); @@ -758,7 +769,10 @@ void SbPlayerBridge::CreatePlayer() { creation_param.output_mode = output_mode_; DCHECK_EQ(sbplayer_interface_->GetPreferredOutputMode(&creation_param), output_mode_); +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StartTimer(MediaTiming::SbPlayerCreate, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL +#if COBALT_MEDIA_ENABLE_PLAYER_SET_MAX_VIDEO_INPUT_SIZE const StarboardExtensionPlayerSetMaxVideoInputSizeApi* player_set_max_video_input_size_extension = static_cast( @@ -771,12 +785,19 @@ void SbPlayerBridge::CreatePlayer() { player_set_max_video_input_size_extension ->SetMaxVideoInputSizeForCurrentThread(max_video_input_size_); } +#endif // COBALT_MEDIA_ENABLE_PLAYER_SET_MAX_VIDEO_INPUT_SIZE player_ = sbplayer_interface_->Create( window_, &creation_param, &SbPlayerBridge::DeallocateSampleCB, &SbPlayerBridge::DecoderStatusCB, &SbPlayerBridge::PlayerStatusCB, &SbPlayerBridge::PlayerErrorCB, this, +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER get_decode_target_graphics_context_provider_func_.Run()); +#else // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER + nullptr); +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StopTimer(MediaTiming::SbPlayerCreate, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL is_creating_player_ = false; @@ -787,21 +808,26 @@ void SbPlayerBridge::CreatePlayer() { if (output_mode_ == kSbPlayerOutputModeDecodeToTexture) { // If the player is setup to decode to texture, then provide Cobalt with // a method of querying that texture. +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER decode_target_provider_->SetGetCurrentSbDecodeTargetFunction(base::Bind( &SbPlayerBridge::GetCurrentSbDecodeTarget, base::Unretained(this))); +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER LOG(INFO) << "Playing in decode-to-texture mode."; } else { LOG(INFO) << "Playing in punch-out mode."; } +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER decode_target_provider_->SetOutputMode( ToVideoFrameProviderOutputMode(output_mode_)); +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER set_bounds_helper_->SetPlayerBridge(this); base::AutoLock auto_lock(lock_); UpdateBounds_Locked(); } +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME void SbPlayerBridge::WriteNextBuffersFromCache(DemuxerStream::Type type, int max_buffers_per_write) { DCHECK(state_ != kSuspended); @@ -842,6 +868,7 @@ void SbPlayerBridge::WriteNextBuffersFromCache(DemuxerStream::Type type, } } } +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME template void SbPlayerBridge::WriteBuffersInternal( @@ -871,10 +898,10 @@ void SbPlayerBridge::WriteBuffersInternal( gathered_sbplayer_sample_infos_subsample_mapping.reserve(buffers.size()); gathered_sbplayer_sample_infos_side_data.reserve(buffers.size()); - for (int i = 0; i < buffers.size(); i++) { + for (int i = 0; i < static_cast(buffers.size()); i++) { const auto& buffer = buffers[i]; if (buffer->end_of_stream()) { - DCHECK_EQ(i, buffers.size() - 1); + DCHECK_EQ(static_cast(i), buffers.size() - 1); if (type == DemuxerStream::AUDIO) { pending_audio_eos_buffer_ = true; } else { @@ -913,8 +940,10 @@ void SbPlayerBridge::WriteBuffersInternal( gathered_sbplayer_sample_infos_side_data.push_back( SbPlayerSampleSideData()); +#if COBALT_MEDIA_ENABLE_SIDE_DATA SbPlayerSampleSideData* side_data = &gathered_sbplayer_sample_infos_side_data[i]; +#endif // COBALT_MEDIA_ENABLE_SIDE_DATA PlayerSampleInfo sample_info = {}; sample_info.type = sample_type; @@ -922,6 +951,7 @@ void SbPlayerBridge::WriteBuffersInternal( sample_info.buffer_size = buffer->data_size(); sample_info.timestamp = buffer->timestamp().InMicroseconds(); +#if COBALT_MEDIA_ENABLE_SIDE_DATA if (buffer->side_data_size() > 0) { // We only support at most one side data currently. side_data->type = kMatroskaBlockAdditional; @@ -930,6 +960,7 @@ void SbPlayerBridge::WriteBuffersInternal( sample_info.side_data = side_data; sample_info.side_data_count = 1; } +#endif // COBALT_MEDIA_ENABLE_SIDE_DATA if (sample_type == kSbMediaTypeAudio) { DCHECK(audio_stream_info); @@ -951,13 +982,17 @@ void SbPlayerBridge::WriteBuffersInternal( } if (!gathered_sbplayer_sample_infos.empty()) { +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StartTimer(MediaTiming::SbPlayerWriteSamples, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL sbplayer_interface_->WriteSamples(player_, sample_type, gathered_sbplayer_sample_infos.data(), gathered_sbplayer_sample_infos.size()); +#if COBALT_MEDIA_ENABLE_CVAL cval_stats_->StopTimer(MediaTiming::SbPlayerWriteSamples, pipeline_identifier_); +#endif // COBALT_MEDIA_ENABLE_CVAL } } @@ -969,8 +1004,8 @@ SbPlayerOutputMode SbPlayerBridge::GetSbPlayerOutputMode() { return output_mode_; } -void SbPlayerBridge::GetInfo_Locked(uint32* video_frames_decoded, - uint32* video_frames_dropped, +void SbPlayerBridge::GetInfo_Locked(uint32_t* video_frames_decoded, + uint32_t* video_frames_dropped, TimeDelta* media_time) { lock_.AssertAcquired(); if (state_ == kSuspended) { @@ -992,7 +1027,7 @@ void SbPlayerBridge::GetInfo_Locked(uint32* video_frames_decoded, sbplayer_interface_->GetInfo(player_, &info); if (media_time) { - *media_time = TimeDelta::FromMicroseconds(info.current_media_timestamp); + *media_time = base::Microseconds(info.current_media_timestamp); } if (video_frames_decoded) { *video_frames_decoded = info.total_video_frames; @@ -1018,6 +1053,7 @@ void SbPlayerBridge::UpdateBounds_Locked() { void SbPlayerBridge::ClearDecoderBufferCache() { DCHECK(task_runner_->RunsTasksInCurrentSequence()); +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME if (state_ != kResuming) { TimeDelta media_time; GetInfo(NULL, NULL, &media_time); @@ -1029,6 +1065,7 @@ void SbPlayerBridge::ClearDecoderBufferCache() { base::Bind(&SbPlayerBridge::CallbackHelper::ClearDecoderBufferCache, callback_helper_), TimeDelta::FromMilliseconds(kClearDecoderCacheIntervalInMilliseconds)); +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME } void SbPlayerBridge::OnDecoderStatus(SbPlayer player, @@ -1067,6 +1104,7 @@ void SbPlayerBridge::OnDecoderStatus(SbPlayer player, auto max_number_of_samples_to_write = SbPlayerGetMaximumNumberOfSamplesPerWrite(player_, type); if (state_ == kResuming) { +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME if (decoder_buffer_cache_.HasMoreBuffers(stream_type)) { WriteNextBuffersFromCache(stream_type, max_number_of_samples_to_write); return; @@ -1075,6 +1113,9 @@ void SbPlayerBridge::OnDecoderStatus(SbPlayer player, !decoder_buffer_cache_.HasMoreBuffers(DemuxerStream::VIDEO)) { state_ = kPlaying; } +#else // COBALT_MEDIA_ENABLE_SUSPEND_RESUME + state_ = kPlaying; +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME } host_->OnNeedData(stream_type, max_number_of_samples_to_write); } @@ -1082,9 +1123,8 @@ void SbPlayerBridge::OnDecoderStatus(SbPlayer player, void SbPlayerBridge::OnPlayerStatus(SbPlayer player, SbPlayerState state, int ticket) { - TRACE_EVENT1("cobalt::media", "SbPlayerBridge::OnPlayerStatus", "state", - state); DCHECK(task_runner_->RunsTasksInCurrentSequence()); + TRACE_EVENT1("media", "SbPlayerBridge::OnPlayerStatus", "state", state); if (player_ != player) { return; @@ -1173,8 +1213,9 @@ void SbPlayerBridge::DecoderStatusCB(SbPlayer player, SbPlayerBridge* helper = static_cast(context); helper->task_runner_->PostTask( FROM_HERE, - base::Bind(&SbPlayerBridge::CallbackHelper::OnDecoderStatus, - helper->callback_helper_, player, type, state, ticket)); + base::BindOnce(&SbPlayerBridge::CallbackHelper::OnDecoderStatus, + helper->callback_helper_, static_cast(player), type, + state, ticket)); } // static @@ -1184,8 +1225,9 @@ void SbPlayerBridge::PlayerStatusCB(SbPlayer player, int ticket) { SbPlayerBridge* helper = static_cast(context); helper->task_runner_->PostTask( - FROM_HERE, base::Bind(&SbPlayerBridge::CallbackHelper::OnPlayerStatus, - helper->callback_helper_, player, state, ticket)); + FROM_HERE, base::BindOnce(&SbPlayerBridge::CallbackHelper::OnPlayerStatus, + helper->callback_helper_, + static_cast(player), state, ticket)); } // static @@ -1202,9 +1244,10 @@ void SbPlayerBridge::PlayerErrorCB(SbPlayer player, } } helper->task_runner_->PostTask( - FROM_HERE, base::Bind(&SbPlayerBridge::CallbackHelper::OnPlayerError, - helper->callback_helper_, player, error, - message ? std::string(message) : "")); + FROM_HERE, + base::BindOnce(&SbPlayerBridge::CallbackHelper::OnPlayerError, + helper->callback_helper_, static_cast(player), + error, message ? std::string(message) : "")); } // static @@ -1213,8 +1256,9 @@ void SbPlayerBridge::DeallocateSampleCB(SbPlayer player, const void* sample_buffer) { SbPlayerBridge* helper = static_cast(context); helper->task_runner_->PostTask( - FROM_HERE, base::Bind(&SbPlayerBridge::CallbackHelper::OnDeallocateSample, - helper->callback_helper_, sample_buffer)); + FROM_HERE, + base::BindOnce(&SbPlayerBridge::CallbackHelper::OnDeallocateSample, + helper->callback_helper_, sample_buffer)); } #if SB_HAS(PLAYER_WITH_URL) @@ -1316,8 +1360,10 @@ void SbPlayerBridge::LogStartupLatency() const { TimeDelta startup_latency = sb_player_state_presenting_time_ - first_event_time; +#if COBALT_MEDIA_ENABLE_STARTUP_LATENCY_TRACKING StatisticsWrapper::GetInstance()->startup_latency.AddSample( startup_latency.InMicroseconds(), 1); +#endif // COBALT_MEDIA_ENABLE_STARTUP_LATENCY_TRACKING // clang-format off LOG(INFO) << FormatString( @@ -1335,10 +1381,17 @@ void SbPlayerBridge::LogStartupLatency() const { first_events_str.c_str(), player_initialization_time_delta.InMicroseconds(), player_preroll_time_delta.InMicroseconds(), first_audio_sample_time_delta.InMicroseconds(), first_video_sample_time_delta.InMicroseconds(), player_presenting_time_delta.InMicroseconds(), +#if COBALT_MEDIA_ENABLE_STARTUP_LATENCY_TRACKING StatisticsWrapper::GetInstance()->startup_latency.min(), StatisticsWrapper::GetInstance()->startup_latency.GetMedian(), StatisticsWrapper::GetInstance()->startup_latency.average(), StatisticsWrapper::GetInstance()->startup_latency.max()); +#else // COBALT_MEDIA_ENABLE_STARTUP_LATENCY_TRACKING + static_cast(0), + static_cast(0), + static_cast(0), + static_cast(0)); +#endif // COBALT_MEDIA_ENABLE_STARTUP_LATENCY_TRACKING // clang-format on } @@ -1361,4 +1414,3 @@ void SbPlayerBridge::SendColorSpaceHistogram() const { } } // namespace media -} // namespace cobalt diff --git a/media/starboard/sbplayer_bridge.h b/media/starboard/sbplayer_bridge.h index c361fea3ff04..c71eed5525b2 100644 --- a/media/starboard/sbplayer_bridge.h +++ b/media/starboard/sbplayer_bridge.h @@ -12,42 +12,46 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef COBALT_MEDIA_BASE_SBPLAYER_BRIDGE_H_ -#define COBALT_MEDIA_BASE_SBPLAYER_BRIDGE_H_ +#ifndef MEDIA_STARBOARD_SBPLAYER_BRIDGE_H_ +#define MEDIA_STARBOARD_SBPLAYER_BRIDGE_H_ #include #include #include #include +#include "base/functional/callback.h" #include "base/memory/ref_counted.h" -#include "base/single_thread_task_runner.h" #include "base/synchronization/lock.h" #include "base/task/sequenced_task_runner.h" +#include "base/task/single_thread_task_runner.h" #include "base/time/time.h" + +#if COBALT_MEDIA_ENABLE_CVAL #include "cobalt/media/base/cval_stats.h" +#endif // COBALT_MEDIA_ENABLE_CVAL + +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER #include "cobalt/media/base/decode_target_provider.h" +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER + +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME #include "cobalt/media/base/decoder_buffer_cache.h" -#include "cobalt/media/base/sbplayer_interface.h" -#include "cobalt/media/base/sbplayer_set_bounds_helper.h" +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME + #include "media/base/audio_decoder_config.h" #include "media/base/decoder_buffer.h" #include "media/base/demuxer_stream.h" #include "media/base/video_decoder_config.h" +#include "media/starboard/sbplayer_interface.h" +#include "media/starboard/sbplayer_set_bounds_helper.h" #include "starboard/media.h" #include "starboard/player.h" #include "ui/gfx/geometry/rect.h" -namespace cobalt { namespace media { -// TODO: Add switch to disable caching class SbPlayerBridge { - typedef ::media::AudioDecoderConfig AudioDecoderConfig; - typedef ::media::DecoderBuffer DecoderBuffer; - typedef ::media::DemuxerStream DemuxerStream; - typedef ::media::VideoDecoderConfig VideoDecoderConfig; - public: class Host { public: @@ -62,7 +66,7 @@ class SbPlayerBridge { }; // Call to get the SbDecodeTargetGraphicsContextProvider for SbPlayerCreate(). - typedef base::Callback + typedef base::RepeatingCallback GetDecodeTargetGraphicsContextProviderFunc; #if SB_HAS(PLAYER_WITH_URL) @@ -97,10 +101,16 @@ class SbPlayerBridge { SbPlayerSetBoundsHelper* set_bounds_helper, bool allow_resume_after_suspend, SbPlayerOutputMode default_output_mode, +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER DecodeTargetProvider* const decode_target_provider, +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER const std::string& max_video_capabilities, - int max_video_input_size, - std::string pipeline_identifier); + int max_video_input_size +#if COBALT_MEDIA_ENABLE_CVAL + , + std::string pipeline_identifier +#endif // COBALT_MEDIA_ENABLE_CVAL + ); ~SbPlayerBridge(); @@ -121,8 +131,8 @@ class SbPlayerBridge { void SetVolume(float volume); void SetPlaybackRate(double playback_rate); - void GetInfo(uint32* video_frames_decoded, - uint32* video_frames_dropped, + void GetInfo(uint32_t* video_frames_decoded, + uint32_t* video_frames_dropped, base::TimeDelta* media_time); std::vector GetAudioConfigurations(); @@ -171,12 +181,14 @@ class SbPlayerBridge { void ClearDecoderBufferCache(); - void OnDecoderStatus(SbPlayer player, + // The following functions accept SbPlayer as void* to work around the + // requirements that types binding to Callbacks have to be complete. + void OnDecoderStatus(void* player, SbMediaType type, SbPlayerDecoderState state, int ticket); - void OnPlayerStatus(SbPlayer player, SbPlayerState state, int ticket); - void OnPlayerError(SbPlayer player, + void OnPlayerStatus(void* player, SbPlayerState state, int ticket); + void OnPlayerError(void* player, SbPlayerError error, const std::string& message); void OnDeallocateSample(const void* sample_buffer); @@ -188,7 +200,7 @@ class SbPlayerBridge { SbPlayerBridge* player_bridge_; }; - static const int64 kClearDecoderCacheIntervalInMilliseconds = 1000; + static const int64_t kClearDecoderCacheIntervalInMilliseconds = 1000; // A map from raw data pointer returned by DecoderBuffer::GetData() to the // DecoderBuffer and a reference count. The reference count indicates how @@ -212,8 +224,10 @@ class SbPlayerBridge { #endif // SB_HAS(PLAYER_WITH_URL) void CreatePlayer(); +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME void WriteNextBuffersFromCache(DemuxerStream::Type type, int max_buffers_per_write); +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME template void WriteBuffersInternal( @@ -222,8 +236,8 @@ class SbPlayerBridge { const SbMediaAudioStreamInfo* audio_stream_info, const SbMediaVideoStreamInfo* video_stream_info); - void GetInfo_Locked(uint32* video_frames_decoded, - uint32* video_frames_dropped, + void GetInfo_Locked(uint32_t* video_frames_decoded, + uint32_t* video_frames_dropped, base::TimeDelta* media_time); void UpdateBounds_Locked(); @@ -280,6 +294,7 @@ class SbPlayerBridge { SbWindow window_; SbDrmSystem drm_system_ = kSbDrmSystemInvalid; Host* const host_; + // TODO(b/376320224): Ensure that set bounds works // Consider merge |SbPlayerSetBoundsHelper| into CallbackHelper. SbPlayerSetBoundsHelper* const set_bounds_helper_; const bool allow_resume_after_suspend_; @@ -297,25 +312,29 @@ class SbPlayerBridge { float volume_ = 1.0f; double playback_rate_ = 0.0; bool seek_pending_ = false; +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME DecoderBufferCache decoder_buffer_cache_; +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME // The following variables can be accessed from GetInfo(), which can be called // from any threads. So some of their usages have to be guarded by |lock_|. base::Lock lock_; // Stores the |z_index| and |rect| parameters of the latest SetBounds() call. - base::Optional set_bounds_z_index_; - base::Optional set_bounds_rect_; + std::optional set_bounds_z_index_; + std::optional set_bounds_rect_; State state_ = kPlaying; SbPlayer player_; - uint32 cached_video_frames_decoded_; - uint32 cached_video_frames_dropped_; + uint32_t cached_video_frames_decoded_; + uint32_t cached_video_frames_dropped_; base::TimeDelta preroll_timestamp_; // Keep track of the output mode we are supposed to output to. SbPlayerOutputMode output_mode_; +#if COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER DecodeTargetProvider* const decode_target_provider_; +#endif // COBALT_MEDIA_ENABLE_DECODE_TARGET_PROVIDER // Keep copies of the mime type strings instead of using the ones in the // DemuxerStreams to ensure that the strings are always valid. @@ -348,11 +367,12 @@ class SbPlayerBridge { bool pending_audio_eos_buffer_ = false; bool pending_video_eos_buffer_ = false; +#if COBALT_MEDIA_ENABLE_CVAL CValStats* cval_stats_; std::string pipeline_identifier_; +#endif // COBALT_MEDIA_ENABLE_CVAL }; } // namespace media -} // namespace cobalt -#endif // COBALT_MEDIA_BASE_SBPLAYER_BRIDGE_H_ +#endif // MEDIA_STARBOARD_SBPLAYER_BRIDGE_H_ diff --git a/media/starboard/sbplayer_interface.cc b/media/starboard/sbplayer_interface.cc index 8db489719785..74f9b9313334 100644 --- a/media/starboard/sbplayer_interface.cc +++ b/media/starboard/sbplayer_interface.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "cobalt/media/base/sbplayer_interface.h" +#include "media/starboard/sbplayer_interface.h" #include @@ -20,7 +20,6 @@ #include "starboard/extension/player_configuration.h" #include "starboard/system.h" -namespace cobalt { namespace media { bool SbPlayerInterface::SetDecodeToTexturePreferred(bool preferred) { @@ -46,23 +45,6 @@ bool SbPlayerInterface::SetDecodeToTexturePreferred(bool preferred) { } } -DefaultSbPlayerInterface::DefaultSbPlayerInterface() { - const CobaltExtensionEnhancedAudioApi* extension_api = - static_cast( - SbSystemGetExtension(kCobaltExtensionEnhancedAudioName)); - if (!extension_api) { - return; - } - - DCHECK_EQ(extension_api->name, - // Avoid comparing raw string pointers for equal. - std::string(kCobaltExtensionEnhancedAudioName)); - DCHECK_EQ(extension_api->version, 1u); - DCHECK_NE(extension_api->PlayerWriteSamples, nullptr); - - enhanced_audio_player_write_samples_ = extension_api->PlayerWriteSamples; -} - SbPlayer DefaultSbPlayerInterface::Create( SbWindow window, const SbPlayerCreationParam* creation_param, @@ -104,30 +86,15 @@ void DefaultSbPlayerInterface::Seek(SbPlayer player, media_metrics_provider_.EndTrackingAction(MediaAction::SBPLAYER_SEEK); } -bool DefaultSbPlayerInterface::IsEnhancedAudioExtensionEnabled() const { - return enhanced_audio_player_write_samples_ != nullptr; -} - void DefaultSbPlayerInterface::WriteSamples( SbPlayer player, SbMediaType sample_type, const SbPlayerSampleInfo* sample_infos, int number_of_sample_infos) { - DCHECK(!IsEnhancedAudioExtensionEnabled()); SbPlayerWriteSamples(player, sample_type, sample_infos, number_of_sample_infos); } -void DefaultSbPlayerInterface::WriteSamples( - SbPlayer player, - SbMediaType sample_type, - const CobaltExtensionEnhancedAudioPlayerSampleInfo* sample_infos, - int number_of_sample_infos) { - DCHECK(IsEnhancedAudioExtensionEnabled()); - enhanced_audio_player_write_samples_(player, sample_type, sample_infos, - number_of_sample_infos); -} - int DefaultSbPlayerInterface::GetMaximumNumberOfSamplesPerWrite( SbPlayer player, SbMediaType sample_type) { @@ -237,4 +204,3 @@ bool DefaultSbPlayerInterface::GetAudioConfiguration( } } // namespace media -} // namespace cobalt diff --git a/media/starboard/sbplayer_interface.h b/media/starboard/sbplayer_interface.h index d29d1c31a9c8..9c6c1e090530 100644 --- a/media/starboard/sbplayer_interface.h +++ b/media/starboard/sbplayer_interface.h @@ -12,20 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef COBALT_MEDIA_BASE_SBPLAYER_INTERFACE_H_ -#define COBALT_MEDIA_BASE_SBPLAYER_INTERFACE_H_ +#ifndef MEDIA_STARBOARD_SBPLAYER_INTERFACE_H_ +#define MEDIA_STARBOARD_SBPLAYER_INTERFACE_H_ #include "base/time/time.h" +#if COBALT_MEDIA_ENABLE_CVAL #include "cobalt/media/base/cval_stats.h" +#endif // COBALT_MEDIA_ENABLE_CVAL +#if COBALT_MEDIA_ENABLE_UMA_METRICS #include "cobalt/media/base/metrics_provider.h" -#include "starboard/extension/enhanced_audio.h" +#endif // COBALT_MEDIA_ENABLE_UMA_METRICS #include "starboard/player.h" #if SB_HAS(PLAYER_WITH_URL) #include SB_URL_PLAYER_INCLUDE_PATH #endif // SB_HAS(PLAYER_WITH_URL) -namespace cobalt { namespace media { class SbPlayerInterface { @@ -48,16 +50,10 @@ class SbPlayerInterface { base::TimeDelta seek_to_timestamp, int ticket) = 0; - virtual bool IsEnhancedAudioExtensionEnabled() const = 0; virtual void WriteSamples(SbPlayer player, SbMediaType sample_type, const SbPlayerSampleInfo* sample_infos, int number_of_sample_infos) = 0; - virtual void WriteSamples( - SbPlayer player, - SbMediaType sample_type, - const CobaltExtensionEnhancedAudioPlayerSampleInfo* sample_infos, - int number_of_sample_infos) = 0; virtual int GetMaximumNumberOfSamplesPerWrite(SbPlayer player, SbMediaType sample_type) = 0; @@ -97,11 +93,42 @@ class SbPlayerInterface { int index, SbMediaAudioConfiguration* out_audio_configuration) = 0; +#if COBALT_MEDIA_ENABLE_CVAL // disabled by default, but can be enabled via h5vcc setting. void EnableCValStats(bool should_enable) { cval_stats_.Enable(should_enable); } CValStats cval_stats_; +#endif // COBALT_MEDIA_ENABLE_CVAL + +#if !COBALT_MEDIA_ENABLE_UMA_METRICS + enum class MediaAction { + UNKNOWN_ACTION, + WEBMEDIAPLAYER_SEEK, + SBPLAYER_CREATE, + SBPLAYER_CREATE_URL_PLAYER, + SBPLAYER_DESTROY, + SBPLAYER_GET_PREFERRED_OUTPUT_MODE, + SBPLAYER_SEEK, + SBPLAYER_WRITE_END_OF_STREAM_AUDIO, + SBPLAYER_WRITE_END_OF_STREAM_VIDEO, + SBPLAYER_SET_BOUNDS, + SBPLAYER_SET_PLAYBACK_RATE, + SBPLAYER_SET_VOLUME, + SBPLAYER_GET_INFO, + SBPLAYER_GET_CURRENT_FRAME, + SBPLAYER_GET_AUDIO_CONFIG, + SBDRM_CREATE, + SBDRM_DESTROY, + SBDRM_GENERATE_SESSION_UPDATE_REQUEST, + SBDRM_UPDATE_SESSION, + SBDRM_CLOSE_SESSION, + }; + struct MediaMetricsProvider { + void StartTrackingAction(...) {} + void EndTrackingAction(...) {} + }; +#endif // !COBALT_MEDIA_ENABLE_UMA_METRICS MediaMetricsProvider media_metrics_provider_; bool SetDecodeToTexturePreferred(bool preferred); @@ -109,8 +136,6 @@ class SbPlayerInterface { class DefaultSbPlayerInterface final : public SbPlayerInterface { public: - DefaultSbPlayerInterface(); - SbPlayer Create( SbWindow window, const SbPlayerCreationParam* creation_param, @@ -126,16 +151,10 @@ class DefaultSbPlayerInterface final : public SbPlayerInterface { void Seek(SbPlayer player, base::TimeDelta seek_to_timestamp, int ticket) override; - bool IsEnhancedAudioExtensionEnabled() const override; void WriteSamples(SbPlayer player, SbMediaType sample_type, const SbPlayerSampleInfo* sample_infos, int number_of_sample_infos) override; - void WriteSamples( - SbPlayer player, - SbMediaType sample_type, - const CobaltExtensionEnhancedAudioPlayerSampleInfo* sample_infos, - int number_of_sample_infos) override; int GetMaximumNumberOfSamplesPerWrite(SbPlayer player, SbMediaType sample_type) override; void WriteEndOfStream(SbPlayer player, SbMediaType stream_type) override; @@ -169,16 +188,8 @@ class DefaultSbPlayerInterface final : public SbPlayerInterface { SbPlayer player, int index, SbMediaAudioConfiguration* out_audio_configuration) override; - - private: - void (*enhanced_audio_player_write_samples_)( - SbPlayer player, - SbMediaType sample_type, - const CobaltExtensionEnhancedAudioPlayerSampleInfo* sample_infos, - int number_of_sample_infos) = nullptr; }; } // namespace media -} // namespace cobalt -#endif // COBALT_MEDIA_BASE_SBPLAYER_INTERFACE_H_ +#endif // MEDIA_STARBOARD_SBPLAYER_INTERFACE_H_ diff --git a/media/starboard/sbplayer_set_bounds_helper.cc b/media/starboard/sbplayer_set_bounds_helper.cc index 9b02952baf72..e49868f946f6 100644 --- a/media/starboard/sbplayer_set_bounds_helper.cc +++ b/media/starboard/sbplayer_set_bounds_helper.cc @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "cobalt/media/base/sbplayer_set_bounds_helper.h" +#include "media/starboard/sbplayer_set_bounds_helper.h" #include "base/atomic_sequence_num.h" -#include "cobalt/media/base/sbplayer_bridge.h" +#include "media/starboard/sbplayer_bridge.h" -namespace cobalt { namespace media { namespace { @@ -47,4 +46,3 @@ bool SbPlayerSetBoundsHelper::SetBounds(int x, int y, int width, int height) { } } // namespace media -} // namespace cobalt diff --git a/media/starboard/sbplayer_set_bounds_helper.h b/media/starboard/sbplayer_set_bounds_helper.h index 4467755038ff..c205d277e917 100644 --- a/media/starboard/sbplayer_set_bounds_helper.h +++ b/media/starboard/sbplayer_set_bounds_helper.h @@ -12,16 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef COBALT_MEDIA_BASE_SBPLAYER_SET_BOUNDS_HELPER_H_ -#define COBALT_MEDIA_BASE_SBPLAYER_SET_BOUNDS_HELPER_H_ +#ifndef MEDIA_STARBOARD_SBPLAYER_SET_BOUNDS_HELPER_H_ +#define MEDIA_STARBOARD_SBPLAYER_SET_BOUNDS_HELPER_H_ + +#include -#include "base/macros.h" #include "base/memory/ref_counted.h" -#include "base/optional.h" #include "base/synchronization/lock.h" #include "ui/gfx/geometry/rect.h" -namespace cobalt { namespace media { class SbPlayerBridge; @@ -37,12 +36,12 @@ class SbPlayerSetBoundsHelper private: base::Lock lock_; SbPlayerBridge* player_bridge_ = nullptr; - base::Optional rect_; + std::optional rect_; - DISALLOW_COPY_AND_ASSIGN(SbPlayerSetBoundsHelper); + SbPlayerSetBoundsHelper(const SbPlayerSetBoundsHelper&) = delete; + void operator=(const SbPlayerSetBoundsHelper&) = delete; }; } // namespace media -} // namespace cobalt -#endif // COBALT_MEDIA_BASE_SBPLAYER_SET_BOUNDS_HELPER_H_ +#endif // MEDIA_STARBOARD_SBPLAYER_SET_BOUNDS_HELPER_H_ diff --git a/media/starboard/starboard_renderer.cc b/media/starboard/starboard_renderer.cc index 279ad43d756a..d07f6ae7c5fc 100644 --- a/media/starboard/starboard_renderer.cc +++ b/media/starboard/starboard_renderer.cc @@ -1,4 +1,4 @@ -// Copyright 2016 The Cobalt Authors. All Rights Reserved. +// Copyright 2024 The Cobalt Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,52 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "cobalt/media/base/sbplayer_pipeline.h" +#include "media/starboard/starboard_renderer.h" -#include -#include - -#include "base/basictypes.h" // For COMPILE_ASSERT -#include "base/bind.h" #include "base/logging.h" -#include "base/strings/stringprintf.h" -#include "base/task/bind_post_task.h" #include "base/trace_event/trace_event.h" -#include "cobalt/base/c_val.h" -#include "cobalt/base/startup_timer.h" -#include "media/base/channel_layout.h" #include "starboard/common/media.h" -#include "starboard/common/string.h" -#include "starboard/configuration_constants.h" -namespace cobalt { namespace media { -namespace { -using ::media::AudioCodec; -using ::media::AudioDecoderConfig; -using ::media::DecoderBuffer; -using ::media::Demuxer; -using ::media::DemuxerHost; -using ::media::DemuxerStream; -using ::media::PipelineStatistics; -using ::media::PipelineStatusCallback; -using ::media::VideoDecoderConfig; -using ::starboard::GetMediaAudioConnectorName; - -static const int kRetryDelayAtSuspendInMilliseconds = 100; - -// In the OnNeedData(), it attempts to write one more audio access -// unit than the audio write duration. Specifically, the check -// |time_ahead_of_playback_for_preroll| > |adjusted_write_duration_for_preroll| -// is used to skip audio writing, using '>' instead of '>='. -// Since the calculated write duration during preroll may align exactly -// with the audio write duration, the current check can fail, leading to an -// additional call to SbPlayerWriteSamples(). By writing an extra guard audio -// buffer, this extra write during preroll can be eliminated. -const int kPrerollGuardAudioBuffer = 1; - -unsigned int g_pipeline_identifier_counter = 0; +namespace { bool HasRemoteAudioOutputs( const std::vector& configurations) { @@ -71,14 +34,14 @@ bool HasRemoteAudioOutputs( case kSbMediaAudioConnectorSpdif: case kSbMediaAudioConnectorUsb: LOG(INFO) << "Encountered local audio connector: " - << GetMediaAudioConnectorName(connector); + << starboard::GetMediaAudioConnectorName(connector); break; case kSbMediaAudioConnectorBluetooth: case kSbMediaAudioConnectorRemoteWired: case kSbMediaAudioConnectorRemoteWireless: case kSbMediaAudioConnectorRemoteOther: LOG(INFO) << "Encountered remote audio connector: " - << GetMediaAudioConnectorName(connector); + << starboard::GetMediaAudioConnectorName(connector); return true; } } @@ -88,742 +51,245 @@ bool HasRemoteAudioOutputs( return false; } -// The function adjusts audio write duration proportionally to the playback -// rate, when the playback rate is greater than 1.0. -// -// Having the right write duration is important: -// 1. Too small of it causes audio underflow. -// 2. Too large of it causes excessive audio switch latency. -// When playback rate is 2x, an 0.5 seconds of write duration effectively only -// lasts for 0.25 seconds and causes audio underflow, and the function will -// adjust it to 1 second in this case. -TimeDelta AdjustWriteDurationForPlaybackRate(TimeDelta write_duration, - float playback_rate) { - if (playback_rate <= 1.0) { - return write_duration; - } - - return write_duration * playback_rate; -} - -// The function returns the default frames per DecoderBuffer. -// -// The number of frames is used to estimate the number of samples per write for -// audio preroll according to |audio_write_duration_|. -int GetDefaultAudioFramesPerBuffer(AudioCodec codec) { - switch (codec) { - case AudioCodec::kOpus: - return 960; - case AudioCodec::kAAC: - return 1024; - case AudioCodec::kAC3: - case AudioCodec::kEAC3: - return 1536; - default: - NOTREACHED(); - return 1; - } -} - } // namespace -SbPlayerPipeline::SbPlayerPipeline( - SbPlayerInterface* interface, PipelineWindow window, - const scoped_refptr& task_runner, - const GetDecodeTargetGraphicsContextProviderFunc& - get_decode_target_graphics_context_provider_func, - bool allow_resume_after_suspend, int max_audio_samples_per_write, - bool force_punch_out_by_default, TimeDelta audio_write_duration_local, - TimeDelta audio_write_duration_remote, MediaLog* media_log, - MediaMetricsProvider* media_metrics_provider, - DecodeTargetProvider* decode_target_provider) - : pipeline_identifier_( - base::StringPrintf("%X", g_pipeline_identifier_counter++)), - sbplayer_interface_(interface), - task_runner_(task_runner), - allow_resume_after_suspend_(allow_resume_after_suspend), - max_audio_samples_per_write_(max_audio_samples_per_write), - window_(window), - get_decode_target_graphics_context_provider_func_( - get_decode_target_graphics_context_provider_func), - natural_size_(0, 0), - volume_(base::StringPrintf("Media.Pipeline.%s.Volume", - pipeline_identifier_.c_str()), - 1.0f, "Volume of the media pipeline."), - playback_rate_(base::StringPrintf("Media.Pipeline.%s.PlaybackRate", - pipeline_identifier_.c_str()), - 0.0f, "Playback rate of the media pipeline."), - duration_(base::StringPrintf("Media.Pipeline.%s.Duration", - pipeline_identifier_.c_str()), - TimeDelta(), "Playback duration of the media pipeline."), - set_bounds_helper_(new SbPlayerSetBoundsHelper), - started_(base::StringPrintf("Media.Pipeline.%s.Started", - pipeline_identifier_.c_str()), - false, "Whether the media pipeline has started."), - suspended_(base::StringPrintf("Media.Pipeline.%s.Suspended", - pipeline_identifier_.c_str()), - false, - "Whether the media pipeline has been suspended. The " - "pipeline is usually suspended after the app enters " - "background mode."), - stopped_(base::StringPrintf("Media.Pipeline.%s.Stopped", - pipeline_identifier_.c_str()), - false, "Whether the media pipeline has stopped."), - ended_(base::StringPrintf("Media.Pipeline.%s.Ended", - pipeline_identifier_.c_str()), - false, "Whether the media pipeline has ended."), - player_state_(base::StringPrintf("Media.Pipeline.%s.PlayerState", - pipeline_identifier_.c_str()), - kSbPlayerStateInitialized, - "The underlying SbPlayer state of the media pipeline."), - decode_target_provider_(decode_target_provider), - audio_write_duration_local_(audio_write_duration_local), - audio_write_duration_remote_(audio_write_duration_remote), - media_metrics_provider_(media_metrics_provider), - last_media_time_(base::StringPrintf("Media.Pipeline.%s.LastMediaTime", - pipeline_identifier_.c_str()), - TimeDelta(), - "Last media time reported by the underlying player."), - max_video_capabilities_( - base::StringPrintf("Media.Pipeline.%s.MaxVideoCapabilities", - pipeline_identifier_.c_str()), - "", "The max video capabilities required for the media pipeline."), - playback_statistics_(pipeline_identifier_) { - if (force_punch_out_by_default) { - default_output_mode_ = kSbPlayerOutputModePunchOut; - } +StarboardRenderer::StarboardRenderer( + const scoped_refptr& task_runner) + : task_runner_(task_runner) { + DCHECK(task_runner_); + LOG(INFO) << "StarboardRenderer constructed."; } -SbPlayerPipeline::~SbPlayerPipeline() { DCHECK(!player_bridge_); } - -void SbPlayerPipeline::Suspend() { - DCHECK(!task_runner_->RunsTasksInCurrentSequence()); - - base::WaitableEvent waitable_event( - base::WaitableEvent::ResetPolicy::MANUAL, - base::WaitableEvent::InitialState::NOT_SIGNALED); - task_runner_->PostTask(FROM_HERE, - base::Bind(&SbPlayerPipeline::SuspendTask, - base::Unretained(this), &waitable_event)); - waitable_event.Wait(); -} - -void SbPlayerPipeline::Resume(PipelineWindow window) { - DCHECK(!task_runner_->RunsTasksInCurrentSequence()); - - base::WaitableEvent waitable_event( - base::WaitableEvent::ResetPolicy::MANUAL, - base::WaitableEvent::InitialState::NOT_SIGNALED); - task_runner_->PostTask( - FROM_HERE, base::Bind(&SbPlayerPipeline::ResumeTask, - base::Unretained(this), window, &waitable_event)); - waitable_event.Wait(); -} - -#if SB_HAS(PLAYER_WITH_URL) -void OnEncryptedMediaInitDataEncountered( - const Pipeline::OnEncryptedMediaInitDataEncounteredCB& - on_encrypted_media_init_data_encountered, - const char* init_data_type, const unsigned char* init_data, - unsigned int init_data_length) { - LOG_IF(WARNING, strcmp(init_data_type, "cenc") != 0 && - strcmp(init_data_type, "fairplay") != 0 && - strcmp(init_data_type, "keyids") != 0 && - strcmp(init_data_type, "webm") != 0) - << "Unknown EME initialization data type: " << init_data_type; - - std::vector init_data_vec(init_data, init_data + init_data_length); - DCHECK(!on_encrypted_media_init_data_encountered.is_null()); - on_encrypted_media_init_data_encountered.Run(init_data_type, init_data_vec); -} -#endif // SB_HAS(PLAYER_WITH_URL) - -void SbPlayerPipeline::Start(Demuxer* demuxer, - const SetDrmSystemReadyCB& set_drm_system_ready_cb, - const PipelineStatusCB& ended_cb, - const ErrorCB& error_cb, const SeekCB& seek_cb, - const BufferingStateCB& buffering_state_cb, - const base::Closure& duration_change_cb, - const base::Closure& output_mode_change_cb, - const base::Closure& content_size_change_cb, - const std::string& max_video_capabilities, - const int max_video_input_size) { - TRACE_EVENT0("cobalt::media", "SbPlayerPipeline::Start"); - - DCHECK(!ended_cb.is_null()); - DCHECK(!error_cb.is_null()); - DCHECK(!seek_cb.is_null()); - DCHECK(!buffering_state_cb.is_null()); - DCHECK(!duration_change_cb.is_null()); - DCHECK(!output_mode_change_cb.is_null()); - DCHECK(!content_size_change_cb.is_null()); - DCHECK(demuxer); - - StartTaskParameters parameters; - parameters.demuxer = demuxer; - parameters.set_drm_system_ready_cb = set_drm_system_ready_cb; - parameters.ended_cb = ended_cb; - parameters.error_cb = error_cb; - parameters.seek_cb = seek_cb; - parameters.buffering_state_cb = buffering_state_cb; - parameters.duration_change_cb = duration_change_cb; - parameters.output_mode_change_cb = output_mode_change_cb; - parameters.content_size_change_cb = content_size_change_cb; - parameters.max_video_capabilities = max_video_capabilities; - parameters.max_video_input_size = max_video_input_size; -#if SB_HAS(PLAYER_WITH_URL) - parameters.is_url_based = false; -#endif // SB_HAS(PLAYER_WITH_URL) - - task_runner_->PostTask(FROM_HERE, - base::Bind(&SbPlayerPipeline::StartTask, this, - base::Passed(¶meters))); -} - -#if SB_HAS(PLAYER_WITH_URL) -void SbPlayerPipeline::Start(const SetDrmSystemReadyCB& set_drm_system_ready_cb, - const OnEncryptedMediaInitDataEncounteredCB& - on_encrypted_media_init_data_encountered_cb, - const std::string& source_url, - const PipelineStatusCB& ended_cb, - const ErrorCB& error_cb, const SeekCB& seek_cb, - const BufferingStateCB& buffering_state_cb, - const base::Closure& duration_change_cb, - const base::Closure& output_mode_change_cb, - const base::Closure& content_size_change_cb) { - TRACE_EVENT0("cobalt::media", "SbPlayerPipeline::Start"); - - DCHECK(!ended_cb.is_null()); - DCHECK(!error_cb.is_null()); - DCHECK(!seek_cb.is_null()); - DCHECK(!buffering_state_cb.is_null()); - DCHECK(!duration_change_cb.is_null()); - DCHECK(!output_mode_change_cb.is_null()); - DCHECK(!content_size_change_cb.is_null()); - DCHECK(!on_encrypted_media_init_data_encountered_cb.is_null()); - - StartTaskParameters parameters; - parameters.demuxer = NULL; - parameters.set_drm_system_ready_cb = set_drm_system_ready_cb; - parameters.ended_cb = ended_cb; - parameters.error_cb = error_cb; - parameters.seek_cb = seek_cb; - parameters.buffering_state_cb = buffering_state_cb; - parameters.duration_change_cb = duration_change_cb; - parameters.output_mode_change_cb = output_mode_change_cb; - parameters.content_size_change_cb = content_size_change_cb; - parameters.source_url = source_url; - parameters.is_url_based = true; - on_encrypted_media_init_data_encountered_cb_ = - base::Bind(&OnEncryptedMediaInitDataEncountered, - on_encrypted_media_init_data_encountered_cb); - set_drm_system_ready_cb_ = parameters.set_drm_system_ready_cb; - DCHECK(!set_drm_system_ready_cb_.is_null()); - RunSetDrmSystemReadyCB(base::Bind(&SbPlayerPipeline::SetDrmSystem, this)); - - task_runner_->PostTask(FROM_HERE, - base::Bind(&SbPlayerPipeline::StartTask, this, - base::Passed(¶meters))); -} -#endif // SB_HAS(PLAYER_WITH_URL) - -void SbPlayerPipeline::Stop(const base::Closure& stop_cb) { - TRACE_EVENT0("cobalt::media", "SbPlayerPipeline::Stop"); - - if (!task_runner_->RunsTasksInCurrentSequence()) { - task_runner_->PostTask(FROM_HERE, - base::Bind(&SbPlayerPipeline::Stop, this, stop_cb)); - return; - } - - DCHECK(stop_cb_.is_null()); - DCHECK(!stop_cb.is_null()); +StarboardRenderer::~StarboardRenderer() { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); - stopped_ = true; + LOG(INFO) << "Destructing StarboardRenderer."; + // Explicitly reset |player_bridge_| before destroying it. + // Some functions in this class using `player_bridge_` can be called + // asynchronously on arbitrary threads (e.g. `GetMediaTime()`), this ensures + // that they won't access `player_bridge_` when it's being destroyed. + decltype(player_bridge_) player_bridge; if (player_bridge_) { - std::unique_ptr player_bridge; - { - base::AutoLock auto_lock(lock_); - player_bridge = std::move(player_bridge_); - } - - LOG(INFO) << "Destroying SbPlayer."; - player_bridge.reset(); - LOG(INFO) << "SbPlayer destroyed."; + base::AutoLock auto_lock(lock_); + player_bridge = std::move(player_bridge_); } + player_bridge.reset(); - // When Stop() is in progress, we no longer need to call |error_cb_|. - error_cb_.Reset(); - if (demuxer_) { - stop_cb_ = stop_cb; - demuxer_->Stop(); - video_stream_ = nullptr; - audio_stream_ = nullptr; - OnDemuxerStopped(); - } else { - stop_cb.Run(); - } + LOG(INFO) << "StarboardRenderer destructed."; } -void SbPlayerPipeline::Seek(TimeDelta time, const SeekCB& seek_cb) { - if (!task_runner_->RunsTasksInCurrentSequence()) { - task_runner_->PostTask( - FROM_HERE, base::Bind(&SbPlayerPipeline::Seek, this, time, seek_cb)); - return; - } - - playback_statistics_.OnSeek(time); - - if (!player_bridge_) { - seek_cb.Run(::media::PIPELINE_ERROR_INVALID_STATE, false, - AppendStatisticsString("SbPlayerPipeline::Seek failed: " - "player_bridge_ is nullptr.")); - return; - } +void StarboardRenderer::Initialize(MediaResource* media_resource, + RendererClient* client, + PipelineStatusCallback init_cb) { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + DCHECK(!client_); + DCHECK(!audio_stream_); + DCHECK(!video_stream_); + DCHECK(media_resource); + DCHECK(client); + DCHECK(init_cb); - player_bridge_->PrepareForSeek(); - ended_ = false; + TRACE_EVENT0("media", "StarboardRenderer::Initialize"); - DCHECK(seek_cb_.is_null()); - DCHECK(!seek_cb.is_null()); + LOG(INFO) << "Initializing StarboardRenderer."; - if (audio_read_in_progress_ || video_read_in_progress_) { - const TimeDelta kDelay = base::Milliseconds(50); +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME + // Note that once this code block is enabled, we should also ensure that the + // posted callback executes on the right object (i.e. not destroyed, use + // WeakPtr?) in the right state (not flushed?). + if (suspended_) { task_runner_->PostDelayedTask( - FROM_HERE, base::Bind(&SbPlayerPipeline::Seek, this, time, seek_cb), - kDelay); - return; - } - - { - base::AutoLock auto_lock(lock_); - seek_cb_ = seek_cb; - seek_time_ = time; - } - - // Ignore pending delayed calls to OnNeedData, and update variables used to - // decide when to delay. - audio_read_delayed_ = false; - StoreMediaTime(seek_time_); - retrograde_media_time_counter_ = 0; - timestamp_of_last_written_audio_ = TimeDelta(); - is_video_eos_written_ = false; - -#if SB_HAS(PLAYER_WITH_URL) - if (is_url_based_) { - player_bridge_->Seek(seek_time_); + FROM_HERE, + base::Bind(&StarboardRenderer::Initialize, this, media_resource, client, + init_cb), + base::Milliseconds(kRetryDelayAtSuspendInMilliseconds)); return; } -#endif // SB_HAS(PLAYER_WITH_URL) - demuxer_->Seek(time, BindPostTaskToCurrentDefault(base::Bind( - &SbPlayerPipeline::OnDemuxerSeeked, this))); -} - -bool SbPlayerPipeline::HasAudio() const { - base::AutoLock auto_lock(lock_); - return audio_stream_ != NULL; -} - -bool SbPlayerPipeline::HasVideo() const { - base::AutoLock auto_lock(lock_); - return video_stream_ != NULL; -} - -float SbPlayerPipeline::GetPlaybackRate() const { - base::AutoLock auto_lock(lock_); - return playback_rate_; -} - -void SbPlayerPipeline::SetPlaybackRate(float playback_rate) { - base::AutoLock auto_lock(lock_); - playback_rate_ = playback_rate; - task_runner_->PostTask( - FROM_HERE, - base::Bind(&SbPlayerPipeline::SetPlaybackRateTask, this, playback_rate)); -} - -float SbPlayerPipeline::GetVolume() const { - base::AutoLock auto_lock(lock_); - return volume_; -} - -void SbPlayerPipeline::SetVolume(float volume) { - if (volume < 0.0f || volume > 1.0f) return; - - base::AutoLock auto_lock(lock_); - volume_ = volume; - task_runner_->PostTask( - FROM_HERE, base::Bind(&SbPlayerPipeline::SetVolumeTask, this, volume)); -} +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME -void SbPlayerPipeline::StoreMediaTime(TimeDelta media_time) { - last_media_time_ = media_time; - last_time_media_time_retrieved_ = Time::Now(); -} + client_ = client; -TimeDelta SbPlayerPipeline::GetMediaTime() { - base::AutoLock auto_lock(lock_); + audio_stream_ = media_resource->GetFirstStream(DemuxerStream::AUDIO); + video_stream_ = media_resource->GetFirstStream(DemuxerStream::VIDEO); - if (!seek_cb_.is_null()) { - StoreMediaTime(seek_time_); - return seek_time_; - } - if (!player_bridge_) { - StoreMediaTime(TimeDelta()); - return TimeDelta(); - } - if (ended_) { - StoreMediaTime(duration_); - return duration_; - } - TimeDelta media_time; -#if SB_HAS(PLAYER_WITH_URL) - if (is_url_based_) { - int frame_width; - int frame_height; - player_bridge_->GetVideoResolution(&frame_width, &frame_height); - if (frame_width != natural_size_.width() || - frame_height != natural_size_.height()) { - natural_size_ = gfx::Size(frame_width, frame_height); - content_size_change_cb_.Run(); - } - } -#endif // SB_HAS(PLAYER_WITH_URL) - player_bridge_->GetInfo(&statistics_.video_frames_decoded, - &statistics_.video_frames_dropped, &media_time); - - // Guarantee that we report monotonically increasing media time - if (media_time < last_media_time_) { - if (retrograde_media_time_counter_ == 0) { - DLOG(WARNING) << "Received retrograde media time, new:" - << media_time.InMicroseconds() - << ", last: " << last_media_time_ << "."; - } - media_time = last_media_time_; - retrograde_media_time_counter_++; - } else if (retrograde_media_time_counter_ != 0) { - DLOG(WARNING) << "Received " << retrograde_media_time_counter_ - << " retrograde media time before recovered."; - retrograde_media_time_counter_ = 0; + if (audio_stream_ == nullptr && video_stream_ == nullptr) { + LOG(INFO) + << "The video has to contain at least an audio track or a video track."; + std::move(init_cb).Run(PipelineStatus( + DEMUXER_ERROR_NO_SUPPORTED_STREAMS, + "The video has to contain at least an audio track or a video track.")); + return; } - StoreMediaTime(media_time); - return media_time; -} -::media::Ranges SbPlayerPipeline::GetBufferedTimeRanges() { +#if COBALT_MEDIA_ENABLE_ENCRYPTED_PLAYBACKS base::AutoLock auto_lock(lock_); -#if SB_HAS(PLAYER_WITH_URL) - ::media::Ranges time_ranges; - - if (!player_bridge_) { - return time_ranges; - } + bool is_encrypted = + audio_stream_ && audio_stream_->audio_decoder_config().is_encrypted(); + is_encrypted |= + video_stream_ && video_stream_->video_decoder_config().is_encrypted(); + if (is_encrypted) { + // TODO(b/328305808): Shall we call client_->OnVideoNaturalSizeChange() to + // provide an initial size before the license exchange finishes? - if (is_url_based_) { - TimeDelta media_time; - TimeDelta buffer_start_time; - TimeDelta buffer_length_time; - player_bridge_->GetInfo(&statistics_.video_frames_decoded, - &statistics_.video_frames_dropped, &media_time); - player_bridge_->GetUrlPlayerBufferedTimeRanges(&buffer_start_time, - &buffer_length_time); - - if (buffer_length_time.InSeconds() == 0) { - buffered_time_ranges_ = time_ranges; - return time_ranges; - } - - time_ranges.Add(buffer_start_time, buffer_start_time + buffer_length_time); - - if (buffered_time_ranges_.size() > 0) { - TimeDelta old_buffer_start_time = buffered_time_ranges_.start(0); - TimeDelta old_buffer_length_time = buffered_time_ranges_.end(0); - int64 old_start_seconds = old_buffer_start_time.InSeconds(); - int64 new_start_seconds = buffer_start_time.InSeconds(); - int64 old_length_seconds = old_buffer_length_time.InSeconds(); - int64 new_length_seconds = buffer_length_time.InSeconds(); - if (old_start_seconds != new_start_seconds || - old_length_seconds != new_length_seconds) { - did_loading_progress_ = true; - } - } else { - did_loading_progress_ = true; - } - - buffered_time_ranges_ = time_ranges; - return time_ranges; + RunSetDrmSystemReadyCB(BindPostTaskToCurrentDefault( + base::Bind(&SbPlayerPipeline::CreatePlayer, this))); + return; } -#endif // SB_HAS(PLAYER_WITH_URL) +#endif // COBALT_MEDIA_ENABLE_ENCRYPTED_PLAYBACKS - return buffered_time_ranges_; + // |init_cb| will be called inside |CreatePlayerBridge()|. + CreatePlayerBridge(std::move(init_cb)); } -TimeDelta SbPlayerPipeline::GetMediaDuration() const { - base::AutoLock auto_lock(lock_); - return duration_; -} - -#if SB_HAS(PLAYER_WITH_URL) -TimeDelta SbPlayerPipeline::GetMediaStartDate() const { - base::AutoLock auto_lock(lock_); - return start_date_; -} -#endif // SB_HAS(PLAYER_WITH_URL) - -void SbPlayerPipeline::GetNaturalVideoSize(gfx::Size* out_size) const { - CHECK(out_size); - base::AutoLock auto_lock(lock_); - *out_size = natural_size_; -} - -std::vector SbPlayerPipeline::GetAudioConnectors() const { - base::AutoLock auto_lock(lock_); - if (!player_bridge_) { - return std::vector(); - } +void StarboardRenderer::Flush(base::OnceClosure flush_cb) { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + DCHECK(!pending_flush_cb_); + DCHECK(flush_cb); - std::vector connectors; + LOG(INFO) << "Flushing StarboardRenderer."; -#if SB_HAS(PLAYER_WITH_URL) - // Url based player does not support audio connectors. - if (is_url_based_) { - return connectors; - } -#endif // SB_HAS(PLAYER_WITH_URL) + // Prepares the |player_bridge_| for Seek(), the |player_bridge_| won't + // request more data from us before Seek() is called. + player_bridge_->PrepareForSeek(); - auto configurations = player_bridge_->GetAudioConfigurations(); - for (auto&& configuration : configurations) { - connectors.push_back(GetMediaAudioConnectorName(configuration.connector)); + // The function can be called when there are in-flight Demuxer::Read() calls + // posted in OnDemuxerStreamRead(), we will wait until they are completed. + if (audio_read_in_progress_ || video_read_in_progress_) { + pending_flush_cb_ = std::move(flush_cb); + } else { + std::move(flush_cb).Run(); } - - return connectors; -} - -bool SbPlayerPipeline::DidLoadingProgress() const { - base::AutoLock auto_lock(lock_); - bool ret = did_loading_progress_; - did_loading_progress_ = false; - return ret; } -PipelineStatistics SbPlayerPipeline::GetStatistics() const { - base::AutoLock auto_lock(lock_); - return statistics_; -} - -Pipeline::SetBoundsCB SbPlayerPipeline::GetSetBoundsCB() { - return base::Bind(&SbPlayerSetBoundsHelper::SetBounds, set_bounds_helper_); -} +void StarboardRenderer::StartPlayingFrom(base::TimeDelta time) { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); -void SbPlayerPipeline::SetPreferredOutputModeToDecodeToTexture() { - TRACE_EVENT0("cobalt::media", - "SbPlayerPipeline::SetPreferredOutputModeToDecodeToTexture"); + LOG(INFO) << "StarboardRenderer::StartPlayingFrom() called with " << time + << '.'; + LOG_IF(WARNING, time < base::Seconds(0)) + << "Potentially invalid start time " << time << '.'; - if (!task_runner_->RunsTasksInCurrentSequence()) { - task_runner_->PostTask( - FROM_HERE, - base::Bind(&SbPlayerPipeline::SetPreferredOutputModeToDecodeToTexture, - this)); + if (player_bridge_initialized_) { + player_bridge_->Seek(time); return; } - // The player can't be created yet, if it is, then we're updating the output - // mode too late. - DCHECK(!player_bridge_); - - default_output_mode_ = kSbPlayerOutputModeDecodeToTexture; + // We cannot start playback before SbPlayerBridge is initialized, save the + // time and start later. + DCHECK(!playing_start_from_time_); + playing_start_from_time_ = time; } -void SbPlayerPipeline::StartTask(StartTaskParameters parameters) { - TRACE_EVENT0("cobalt::media", "SbPlayerPipeline::StartTask"); - +void StarboardRenderer::SetPlaybackRate(double playback_rate) { DCHECK(task_runner_->RunsTasksInCurrentSequence()); - DCHECK(!demuxer_); - - demuxer_ = parameters.demuxer; - set_drm_system_ready_cb_ = parameters.set_drm_system_ready_cb; - ended_cb_ = parameters.ended_cb; - error_cb_ = parameters.error_cb; - { - base::AutoLock auto_lock(lock_); - seek_cb_ = parameters.seek_cb; - } - buffering_state_cb_ = parameters.buffering_state_cb; - duration_change_cb_ = parameters.duration_change_cb; - output_mode_change_cb_ = parameters.output_mode_change_cb; - content_size_change_cb_ = parameters.content_size_change_cb; - max_video_capabilities_ = parameters.max_video_capabilities; - max_video_input_size_ = parameters.max_video_input_size; -#if SB_HAS(PLAYER_WITH_URL) - is_url_based_ = parameters.is_url_based; - if (is_url_based_) { - CreateUrlPlayer(parameters.source_url); + if (playback_rate < 0.0) { + LOG(INFO) << "StarboardRenderer::SetPlaybackRate(): invalid playback rate " + << playback_rate << '.'; return; } -#endif // SB_HAS(PLAYER_WITH_URL) - demuxer_->Initialize( - this, BindPostTaskToCurrentDefault( - base::Bind(&SbPlayerPipeline::OnDemuxerInitialized, this))); - - started_ = true; -} -void SbPlayerPipeline::SetVolumeTask(float volume) { - DCHECK(task_runner_->RunsTasksInCurrentSequence()); + LOG(INFO) << "StarboardRenderer changes playback rate from " << playback_rate_ + << " to " << playback_rate << '.'; - if (player_bridge_) { - player_bridge_->SetVolume(volume_); + if (playback_rate_ == playback_rate) { + return; } -} -void SbPlayerPipeline::SetPlaybackRateTask(float volume) { - DCHECK(task_runner_->RunsTasksInCurrentSequence()); + playback_rate_ = playback_rate; if (player_bridge_) { player_bridge_->SetPlaybackRate(playback_rate_); } } -void SbPlayerPipeline::SetDurationTask(TimeDelta duration) { +void StarboardRenderer::SetVolume(float volume) { DCHECK(task_runner_->RunsTasksInCurrentSequence()); - if (!duration_change_cb_.is_null()) { - duration_change_cb_.Run(); - } -} -void SbPlayerPipeline::OnBufferedTimeRangesChanged( - const ::media::Ranges& ranges) { - base::AutoLock auto_lock(lock_); - did_loading_progress_ = true; - buffered_time_ranges_ = ranges; -} - -void SbPlayerPipeline::SetDuration(TimeDelta duration) { - base::AutoLock auto_lock(lock_); - duration_ = duration; - task_runner_->PostTask( - FROM_HERE, - base::Bind(&SbPlayerPipeline::SetDurationTask, this, duration)); -} - -void SbPlayerPipeline::OnDemuxerError(PipelineStatus error) { - if (!task_runner_->RunsTasksInCurrentSequence()) { - task_runner_->PostTask( - FROM_HERE, base::Bind(&SbPlayerPipeline::OnDemuxerError, this, error)); + if (volume < 0.0 || volume > 1.0) { + LOG(INFO) << "StarboardRenderer::SetVolume(): invalid volume " << volume + << '.'; return; } - LOG(INFO) << "SbPlayerPipeline::OnDemuxerError() called with error " << error; + LOG(INFO) << "StarboardRenderer changes volume from " << volume_ << " to " + << volume << '.'; - if (error != ::media::PIPELINE_OK) { - CallErrorCB(error, "Demuxer error."); + if (volume_ == volume) { + return; } -} -#if SB_HAS(PLAYER_WITH_URL) -void SbPlayerPipeline::CreateUrlPlayer(const std::string& source_url) { - TRACE_EVENT0("cobalt::media", "SbPlayerPipeline::CreateUrlPlayer"); - DCHECK(task_runner_->RunsTasksInCurrentSequence()); + volume_ = volume; - if (stopped_) { - return; + if (player_bridge_) { + player_bridge_->SetVolume(volume_); } +} - if (suspended_) { - task_runner_->PostDelayedTask( - FROM_HERE, - base::Bind(&SbPlayerPipeline::CreateUrlPlayer, this, source_url), - TimeDelta::FromMilliseconds(kRetryDelayAtSuspendInMilliseconds)); - return; - } +// TODO(b/376328722): Revisit playback time reporting. +base::TimeDelta StarboardRenderer::GetMediaTime() { + base::AutoLock auto_lock(lock_); - // TODO: Check |suspended_| here as the pipeline can be suspended before the - // player is created. In this case we should delay creating the player as the - // creation of player may fail. - std::string error_message; - { - base::AutoLock auto_lock(lock_); - LOG(INFO) << "Creating SbPlayerBridge with url: " << source_url; - player_bridge_.reset(new SbPlayerBridge( - sbplayer_interface_, task_runner_, source_url, window_, this, - set_bounds_helper_.get(), allow_resume_after_suspend_, - default_output_mode_, on_encrypted_media_init_data_encountered_cb_, - decode_target_provider_, pipeline_identifier_)); - if (player_bridge_->IsValid()) { - SetPlaybackRateTask(playback_rate_); - SetVolumeTask(volume_); - } else { - error_message = player_bridge_->GetPlayerCreationErrorMessage(); - player_bridge_.reset(); - LOG(INFO) << "Failed to create a valid SbPlayerBridge."; - } + if (!player_bridge_) { + return base::Microseconds(0); } - if (player_bridge_ && player_bridge_->IsValid()) { - base::Closure output_mode_change_cb; - { - base::AutoLock auto_lock(lock_); - DCHECK(!output_mode_change_cb_.is_null()); - output_mode_change_cb = std::move(output_mode_change_cb_); - } - output_mode_change_cb.Run(); - return; - } + uint32_t video_frames_decoded, video_frames_dropped; + base::TimeDelta media_time; - std::string time_information = GetTimeInformation(); - LOG(INFO) << "SbPlayerPipeline::CreateUrlPlayer failed to create a valid " - "SbPlayerBridge - " - << time_information << " \'" << error_message << "\'"; + player_bridge_->GetInfo(&video_frames_decoded, &video_frames_dropped, + &media_time); - CallSeekCB(::media::DECODER_ERROR_NOT_SUPPORTED, - "SbPlayerPipeline::CreateUrlPlayer failed to create a valid " - "SbPlayerBridge - " + - time_information + " \'" + error_message + "\'"); -} + // Report dropped frames since we have the info anyway. + PipelineStatistics statistics; -void SbPlayerPipeline::SetDrmSystem(SbDrmSystem drm_system) { - TRACE_EVENT0("cobalt::media", "SbPlayerPipeline::SetDrmSystem"); + if (video_frames_decoded > last_video_frames_decoded_) { + statistics.video_frames_decoded = + video_frames_decoded - last_video_frames_decoded_; + last_video_frames_decoded_ = video_frames_decoded; + } - base::AutoLock auto_lock(lock_); - if (!player_bridge_) { - LOG(INFO) << "Player not set before calling SbPlayerPipeline::SetDrmSystem"; - return; + if (video_frames_dropped > last_video_frames_dropped_) { + statistics.video_frames_dropped = + video_frames_dropped - last_video_frames_dropped_; + last_video_frames_dropped_ = video_frames_dropped; } - if (player_bridge_->IsValid()) { - player_bridge_->RecordSetDrmSystemReadyTime(set_drm_system_ready_cb_time_); - player_bridge_->SetDrmSystem(drm_system); + if (statistics.video_frames_decoded > 0 || + statistics.video_frames_dropped > 0) { + // TODO(b/375273093): Refine frame drops reporting, and double check + // whether we should report more statistics, and/or + // reduce the frequency of reporting. + task_runner_->PostTask( + FROM_HERE, base::BindOnce(&RendererClient::OnStatisticsUpdate, + base::Unretained(client_), statistics)); } -} -#endif // SB_HAS(PLAYER_WITH_URL) -void SbPlayerPipeline::CreatePlayer(SbDrmSystem drm_system) { -#if SB_HAS(PLAYER_WITH_URL) - DCHECK(!is_url_based_); -#endif // SB_HAS(PLAYER_WITH_URL - TRACE_EVENT0("cobalt::media", "SbPlayerPipeline::CreatePlayer"); + return media_time; +} +void StarboardRenderer::CreatePlayerBridge(PipelineStatusCallback init_cb) { DCHECK(task_runner_->RunsTasksInCurrentSequence()); + DCHECK(init_cb); DCHECK(audio_stream_ || video_stream_); - if (stopped_) { - return; - } + TRACE_EVENT0("media", "StarboardRenderer::CreatePlayerBridge"); +#if COBALT_MEDIA_ENABLE_SUSPEND_RESUME + // Note that once this code block is enabled, we should also ensure that the + // posted callback executes on the right object (i.e. not destroyed, use + // WeakPtr?) in the right state (not flushed?). if (suspended_) { task_runner_->PostDelayedTask( FROM_HERE, - base::Bind(&SbPlayerPipeline::CreatePlayer, this, drm_system), + base::Bind(&StarboardRenderer::CreatePlayerBridge, this, init_cb), base::Milliseconds(kRetryDelayAtSuspendInMilliseconds)); return; } +#endif // COBALT_MEDIA_ENABLE_SUSPEND_RESUME - // TODO: Check |suspended_| here as the pipeline can be suspended before the - // player is created. In this case we should delay creating the player as the - // creation of player may fail. AudioDecoderConfig invalid_audio_config; const AudioDecoderConfig& audio_config = audio_stream_ ? audio_stream_->audio_decoder_config() @@ -833,43 +299,64 @@ void SbPlayerPipeline::CreatePlayer(SbDrmSystem drm_system) { video_stream_ ? video_stream_->video_decoder_config() : invalid_video_config; - std::string audio_mime_type = audio_stream_ ? audio_stream_->mime_type() : ""; - std::string video_mime_type; - if (video_stream_) { - playback_statistics_.UpdateVideoConfig( - video_stream_->video_decoder_config()); - video_mime_type = video_stream_->mime_type(); - } + std::string audio_mime_type = ""; + std::string video_mime_type = ""; + + // TODO(b/321842876): Enable mime type passing in //media. + // + // audio_mime_type = audio_stream_ ? audio_stream_->mime_type() : ""; + // if (video_stream_) { + // video_mime_type = video_stream_->mime_type(); + // } std::string error_message; + { base::AutoLock auto_lock(lock_); - SB_DCHECK(!player_bridge_); - // In the extreme case that CreatePlayer() is called when a |player_bridge_| - // is available, reset the existing player first to reduce the number of - // active players. + DCHECK(!player_bridge_); + + // In the unexpected case that CreatePlayerBridge() is called when a + // |player_bridge_| is set, reset the existing player first to reduce the + // number of active players. player_bridge_.reset(); + LOG(INFO) << "Creating SbPlayerBridge."; + player_bridge_.reset(new SbPlayerBridge( - sbplayer_interface_, task_runner_, - get_decode_target_graphics_context_provider_func_, audio_config, - audio_mime_type, video_config, video_mime_type, window_, drm_system, - this, set_bounds_helper_.get(), allow_resume_after_suspend_, - default_output_mode_, decode_target_provider_, max_video_capabilities_, - max_video_input_size_, pipeline_identifier_)); + &sbplayer_interface_, task_runner_, + // TODO(b/375070492): Implement decode-to-texture support + SbPlayerBridge::GetDecodeTargetGraphicsContextProviderFunc(), + audio_config, + // TODO(b/321842876): Enable mime type passing in //media. + audio_mime_type, video_config, + // TODO(b/321842876): Enable mime type passing in //media. + video_mime_type, + // TODO(b/326497953): Support suspend/resume. + // TODO(b/326508279): Support background mode. + kSbWindowInvalid, + // TODO(b/328305808): Implement SbDrm support. + kSbDrmSystemInvalid, this, + // TODO(b/376320224); Verify set bounds works + nullptr, + // TODO(b/326497953): Support suspend/resume. + false, + // TODO(b/326825450): Revisit 360 videos. + // TODO(b/326827007): Support secondary videos. + kSbPlayerOutputModeInvalid, + // TODO(b/326827007): Support secondary videos. + "", + // TODO(b/326654546): Revisit HTMLVideoElement.setMaxVideoInputSize. + -1)); if (player_bridge_->IsValid()) { // TODO(b/267678497): When `player_bridge_->GetAudioConfigurations()` // returns no audio configurations, update the write durations again // before the SbPlayer reaches `kSbPlayerStatePresenting`. - audio_write_duration_for_preroll_ = audio_write_duration_ = + audio_write_duration_ = HasRemoteAudioOutputs(player_bridge_->GetAudioConfigurations()) ? audio_write_duration_remote_ : audio_write_duration_local_; LOG(INFO) << "SbPlayerBridge created, with audio write duration at " - << audio_write_duration_for_preroll_; - - SetPlaybackRateTask(playback_rate_); - SetVolumeTask(volume_); + << audio_write_duration_; } else { error_message = player_bridge_->GetPlayerCreationErrorMessage(); player_bridge_.reset(); @@ -878,218 +365,131 @@ void SbPlayerPipeline::CreatePlayer(SbDrmSystem drm_system) { } if (player_bridge_ && player_bridge_->IsValid()) { - player_bridge_->RecordSetDrmSystemReadyTime(set_drm_system_ready_cb_time_); - base::Closure output_mode_change_cb; - { - base::AutoLock auto_lock(lock_); - DCHECK(!output_mode_change_cb_.is_null()); - output_mode_change_cb = std::move(output_mode_change_cb_); - } - output_mode_change_cb.Run(); - if (audio_stream_) { UpdateDecoderConfig(audio_stream_); } if (video_stream_) { UpdateDecoderConfig(video_stream_); } + + player_bridge_->SetPlaybackRate(playback_rate_); + player_bridge_->SetVolume(volume_); + + std::move(init_cb).Run(PipelineStatus(PIPELINE_OK)); return; } - std::string time_information = GetTimeInformation(); - LOG(INFO) << "SbPlayerPipeline::CreatePlayer failed to create a valid " - "SbPlayerBridge - " - << time_information << " \'" << error_message << "\'"; + LOG(INFO) << "SbPlayerPipeline::CreatePlayerBridge() failed to create a" + " valid SbPlayerBridge - \"" + << error_message << "\""; - CallSeekCB(::media::DECODER_ERROR_NOT_SUPPORTED, - "SbPlayerPipeline::CreatePlayer failed to create a valid " - "SbPlayerBridge - " + - time_information + " \'" + error_message + "\'"); + std::move(init_cb).Run(PipelineStatus( + DECODER_ERROR_NOT_SUPPORTED, + "SbPlayerPipeline::CreatePlayerBridge() failed to create a valid" + " SbPlayerBridge - \"" + + error_message + "\"")); } -void SbPlayerPipeline::OnDemuxerInitialized(PipelineStatus status) { -#if SB_HAS(PLAYER_WITH_URL) - DCHECK(!is_url_based_); -#endif // SB_HAS(PLAYER_WITH_URL) - TRACE_EVENT0("cobalt::media", "SbPlayerPipeline::OnDemuxerInitialized"); - +void StarboardRenderer::UpdateDecoderConfig(DemuxerStream* stream) { DCHECK(task_runner_->RunsTasksInCurrentSequence()); - if (stopped_) { + if (!player_bridge_) { return; } - if (status != ::media::PIPELINE_OK) { - CallErrorCB(status, "Demuxer initialization error."); - return; - } + if (stream->type() == DemuxerStream::AUDIO) { + const AudioDecoderConfig& decoder_config = stream->audio_decoder_config(); + player_bridge_->UpdateAudioConfig(decoder_config, ""); + } else { + DCHECK_EQ(stream->type(), DemuxerStream::VIDEO); + const VideoDecoderConfig& decoder_config = stream->video_decoder_config(); - if (suspended_) { - task_runner_->PostDelayedTask( - FROM_HERE, - base::Bind(&SbPlayerPipeline::OnDemuxerInitialized, this, status), - base::Milliseconds(kRetryDelayAtSuspendInMilliseconds)); - return; - } + // TODO(b/375275033): Refine natural size change handling. +#if 0 + base::AutoLock auto_lock(lock_); - auto streams = demuxer_->GetAllStreams(); - DemuxerStream* audio_stream = nullptr; - DemuxerStream* video_stream = nullptr; - for (auto&& stream : streams) { - if (stream->type() == DemuxerStream::AUDIO) { - if (audio_stream == nullptr) { - audio_stream = stream; - } else { - LOG(WARNING) << "Encountered more than one audio streams."; - } - } else if (stream->type() == DemuxerStream::VIDEO) { - if (video_stream == nullptr) { - video_stream = stream; - } else { - LOG(WARNING) << "Encountered more than one video streams."; - } - } - } + bool natural_size_changed = + (decoder_config.natural_size().width() != natural_size_.width() || + decoder_config.natural_size().height() != natural_size_.height()); - if (audio_stream == NULL && video_stream == NULL) { - LOG(INFO) << "The video has to contain an audio track or a video track."; - CallErrorCB(::media::DEMUXER_ERROR_NO_SUPPORTED_STREAMS, - "The video has to contain an audio track or a video track."); - return; - } + natural_size_ = decoder_config.natural_size(); + + player_bridge_->UpdateVideoConfig(decoder_config, ""); - { - base::AutoLock auto_lock(lock_); - audio_stream_ = audio_stream; - video_stream_ = video_stream; - - bool is_encrypted = - audio_stream_ && audio_stream_->audio_decoder_config().is_encrypted(); - is_encrypted |= - video_stream_ && video_stream_->video_decoder_config().is_encrypted(); - bool natural_size_changed = false; - if (video_stream_) { - natural_size_changed = - (video_stream_->video_decoder_config().natural_size().width() != - natural_size_.width() || - video_stream_->video_decoder_config().natural_size().height() != - natural_size_.height()); - natural_size_ = video_stream_->video_decoder_config().natural_size(); - } if (natural_size_changed) { content_size_change_cb_.Run(); } - if (is_encrypted) { - RunSetDrmSystemReadyCB(BindPostTaskToCurrentDefault( - base::Bind(&SbPlayerPipeline::CreatePlayer, this))); - return; - } - } - - CreatePlayer(kSbDrmSystemInvalid); -} - -void SbPlayerPipeline::OnDemuxerSeeked(PipelineStatus status) { - DCHECK(task_runner_->RunsTasksInCurrentSequence()); - - if (status == ::media::PIPELINE_OK && player_bridge_) { - player_bridge_->Seek(seek_time_); +#endif // 0 } } -void SbPlayerPipeline::OnDemuxerStopped() { - TRACE_EVENT0("cobalt::media", "SbPlayerPipeline::OnDemuxerStopped"); - - if (!task_runner_->RunsTasksInCurrentSequence()) { - task_runner_->PostTask( - FROM_HERE, base::Bind(&SbPlayerPipeline::OnDemuxerStopped, this)); - return; - } - - std::move(stop_cb_).Run(); -} - -void SbPlayerPipeline::OnDemuxerStreamRead( - DemuxerStream::Type type, int max_number_buffers_to_read, +void StarboardRenderer::OnDemuxerStreamRead( + DemuxerStream* stream, DemuxerStream::Status status, - const std::vector> buffers) { -#if SB_HAS(PLAYER_WITH_URL) - DCHECK(!is_url_based_); -#endif // SB_HAS(PLAYER_WITH_URL) - DCHECK(type == DemuxerStream::AUDIO || type == DemuxerStream::VIDEO) - << "Unsupported DemuxerStream::Type " << type; - + DemuxerStream::DecoderBufferVector buffers) { if (!task_runner_->RunsTasksInCurrentSequence()) { task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&SbPlayerPipeline::OnDemuxerStreamRead, this, type, - max_number_buffers_to_read, status, buffers)); + FROM_HERE, base::BindOnce(&StarboardRenderer::OnDemuxerStreamRead, + weak_this_, stream, status, buffers)); return; } - if (stopped_) { - return; - } - - DCHECK(player_bridge_); - - DemuxerStream* stream = - type == DemuxerStream::AUDIO ? audio_stream_ : video_stream_; - DCHECK(stream); - - if (!player_bridge_ || !stream) { - return; - } - - if (status == DemuxerStream::kAborted) { - DCHECK(GetReadInProgress(type)); - SetReadInProgress(type, false); - - if (!seek_cb_.is_null()) { - CallSeekCB(::media::PIPELINE_OK, ""); + if (pending_flush_cb_) { + if (stream == audio_stream_) { + DCHECK(audio_read_in_progress_); + audio_read_in_progress_ = false; + } + if (stream == video_stream_) { + DCHECK(video_read_in_progress_); + video_read_in_progress_ = false; + } + if (!audio_read_in_progress_ && !video_read_in_progress_) { + auto flush_cb = std::move(pending_flush_cb_); + std::move(flush_cb).Run(); } return; } - if (status == DemuxerStream::kConfigChanged) { - UpdateDecoderConfig(stream); - stream->Read(max_number_buffers_to_read, - base::BindOnce(&SbPlayerPipeline::OnDemuxerStreamRead, this, - type, max_number_buffers_to_read)); - return; - } + DCHECK(player_bridge_); - if (type == DemuxerStream::AUDIO) { - for (const auto& buffer : buffers) { - playback_statistics_.OnAudioAU(buffer); - if (!buffer->end_of_stream()) { - last_audio_sample_interval_ = - buffer->timestamp() - timestamp_of_last_written_audio_; - timestamp_of_last_written_audio_ = buffer->timestamp(); - } + if (status == DemuxerStream::kOk) { + if (stream == audio_stream_) { + DCHECK(audio_read_in_progress_); + audio_read_in_progress_ = false; + player_bridge_->WriteBuffers(DemuxerStream::AUDIO, buffers); + } else { + DCHECK(video_read_in_progress_); + video_read_in_progress_ = false; + player_bridge_->WriteBuffers(DemuxerStream::VIDEO, buffers); } - } else { - for (const auto& buffer : buffers) { - playback_statistics_.OnVideoAU(buffer); - if (buffer->end_of_stream()) { - is_video_eos_written_ = true; - } + } else if (status == DemuxerStream::kAborted) { + } else if (status == DemuxerStream::kConfigChanged) { + if (stream == audio_stream_) { + client_->OnAudioConfigChange(stream->audio_decoder_config()); + } else { + DCHECK_EQ(stream, video_stream_); + client_->OnVideoConfigChange(stream->video_decoder_config()); + // TODO(b/375275033): Refine calling to OnVideoNaturalSizeChange(). + client_->OnVideoNaturalSizeChange( + stream->video_decoder_config().visible_rect().size()); } + UpdateDecoderConfig(stream); + stream->Read(1, base::BindOnce(&StarboardRenderer::OnDemuxerStreamRead, + weak_this_, stream)); + } else if (status == DemuxerStream::kError) { + client_->OnError(PIPELINE_ERROR_READ); } - SetReadInProgress(type, false); - - player_bridge_->WriteBuffers(type, buffers); } -void SbPlayerPipeline::OnNeedData(DemuxerStream::Type type, - int max_number_of_buffers_to_write) { -#if SB_HAS(PLAYER_WITH_URL) - DCHECK(!is_url_based_); -#endif // SB_HAS(PLAYER_WITH_URL) +void StarboardRenderer::OnNeedData(DemuxerStream::Type type, + int max_number_of_buffers_to_write) { DCHECK(task_runner_->RunsTasksInCurrentSequence()); - // In case if Stop() has been called. + // In case if the callback is fired when creation of the `player_bridge_` + // fails. + // We may also need this for suspend/resume support. if (!player_bridge_) { + LOG(INFO) << "StarboardRenderer::OnNeedData() called with " << type; return; } @@ -1098,15 +498,20 @@ void SbPlayerPipeline::OnNeedData(DemuxerStream::Type type, max_audio_samples_per_write_) : 1; - if (GetReadInProgress(type)) return; - if (type == DemuxerStream::AUDIO) { if (!audio_stream_) { LOG(WARNING) - << "Calling OnNeedData() for audio data during audioless playback"; + << "Calling OnNeedData() for audio data during audioless playback."; return; } + if (audio_read_in_progress_) { + LOG(WARNING) + << "Calling OnNeedData() when there is an audio read in progress."; + return; + } + +#if COBALT_MEDIA_ENABLE_AUDIO_DURATION // If we haven't checked the media time recently, update it now. if (Time::Now() - last_time_media_time_retrieved_ > kMediaTimeCheckInterval) { @@ -1117,7 +522,7 @@ void SbPlayerPipeline::OnNeedData(DemuxerStream::Type type, // after the player has received enough audio for preroll, taking into // account that our estimate of playback time might be behind by // |kMediaTimeCheckInterval|. - TimeDelta time_ahead_of_playback_for_preroll = + base::TimeDelta time_ahead_of_playback_for_preroll = timestamp_of_last_written_audio_ - seek_time_; auto adjusted_write_duration_for_preroll = AdjustWriteDurationForPlaybackRate(audio_write_duration_for_preroll_, @@ -1172,391 +577,82 @@ void SbPlayerPipeline::OnNeedData(DemuxerStream::Type type, max_buffers = std::min(max_buffers, estimated_max_buffers); audio_read_delayed_ = false; +#endif // COBALT_MEDIA_ENABLE_AUDIO_DURATION audio_read_in_progress_ = true; } else { DCHECK_EQ(type, DemuxerStream::VIDEO); + + if (video_read_in_progress_) { + LOG(WARNING) + << "Calling OnNeedData() when there is a video read in progress."; + return; + } + video_read_in_progress_ = true; } - DemuxerStream* stream = - type == DemuxerStream::AUDIO ? audio_stream_ : video_stream_; + auto stream = (type == DemuxerStream::AUDIO ? audio_stream_ : video_stream_); DCHECK(stream); stream->Read(max_buffers, - base::BindOnce(&SbPlayerPipeline::OnDemuxerStreamRead, this, - type, max_buffers)); -} - -int SbPlayerPipeline::GetDefaultMaxBuffers(AudioCodec codec, - TimeDelta duration_to_write, - bool is_preroll) { - // Return default maximum samples per write to speed up the initial sample - // write, including guard number of samples per write for audio preroll. - // The guard number kPrerollGuardAudioBuffer is used to ensure Cobalt - // can do one initial write for audio preroll. - int default_max_buffers = static_cast( - std::ceil(duration_to_write.InSecondsF() * - audio_stream_->audio_decoder_config().samples_per_second() / - GetDefaultAudioFramesPerBuffer(codec))); - if (is_preroll) { - default_max_buffers += kPrerollGuardAudioBuffer; - } - DCHECK_GT(default_max_buffers, 0); - return default_max_buffers; -} - -int SbPlayerPipeline::GetEstimatedMaxBuffers(TimeDelta write_duration, - TimeDelta time_ahead_of_playback, - bool is_preroll) { - DCHECK_GE(time_ahead_of_playback.InMicroseconds(), 0); - - int estimated_max_buffers = 1; - if (!(max_audio_samples_per_write_ > 1) || - write_duration <= time_ahead_of_playback) { - return estimated_max_buffers; - } - - TimeDelta duration_to_write = write_duration - time_ahead_of_playback; - DCHECK_GT(duration_to_write.InMicroseconds(), 0); - switch (audio_stream_->audio_decoder_config().codec()) { - case AudioCodec::kOpus: - case AudioCodec::kAAC: - case AudioCodec::kAC3: - case AudioCodec::kEAC3: - if (last_audio_sample_interval_.is_zero()) { - estimated_max_buffers = - GetDefaultMaxBuffers(audio_stream_->audio_decoder_config().codec(), - duration_to_write, is_preroll); - break; - } - // TODO(b/41486346): Support multiple samples per write on the format IAMF. - case AudioCodec::kIAMF: - default: - if (!last_audio_sample_interval_.is_zero()) { - DCHECK_GT(last_audio_sample_interval_.InMicroseconds(), 0); - estimated_max_buffers = - duration_to_write.InMillisecondsRoundedUp() / - last_audio_sample_interval_.InMilliseconds() + - 1; - } - } - DCHECK_GT(estimated_max_buffers, 0); - // Return 1 if |estimated_max_buffers| is non-positive. This ensures - // in corner cases, |estimated_max_buffers| falls back to 1. - // The maximum number samples of write should be guarded by - // SbPlayerGetMaximumNumberOfSamplesPerWrite() in OnNeedData(). - return estimated_max_buffers > 0 ? estimated_max_buffers : 1; + base::BindOnce(&StarboardRenderer::OnDemuxerStreamRead, + weak_this_, stream)); } -void SbPlayerPipeline::OnPlayerStatus(SbPlayerState state) { +void StarboardRenderer::OnPlayerStatus(SbPlayerState state) { DCHECK(task_runner_->RunsTasksInCurrentSequence()); - // In case if Stop() has been called. + // In case if state is changed when creation of the `player_bridge_` fails. + // We may also need this for suspend/resume support. if (!player_bridge_) { + // TODO(b/376316272): Turn `state` into its string form and consolidate all + // player state loggings below. + LOG(INFO) << "StarboardRenderer::OnPlayerStatus() called with " << state; return; } - player_state_ = state; + switch (state) { case kSbPlayerStateInitialized: - NOTREACHED(); + LOG(INFO) << "StarboardRenderer::OnPlayerStatus() called with " + "kSbPlayerStateInitialized."; + DCHECK(!player_bridge_initialized_); + player_bridge_initialized_ = true; + + // TODO(b/376317639): Revisit the handling of StartPlayingFrom() called + // before player enters kSbPlayerStateInitialized. + if (playing_start_from_time_) { + StartPlayingFrom(std::move(playing_start_from_time_).value()); + } break; case kSbPlayerStatePrerolling: -#if SB_HAS(PLAYER_WITH_URL) - if (is_url_based_) { - break; - } -#endif // SB_HAS(PLAYER_WITH_URL) - buffering_state_cb_.Run(kHaveMetadata); + LOG(INFO) << "StarboardRenderer::OnPlayerStatus() called with " + "kSbPlayerStatePrerolling."; break; - case kSbPlayerStatePresenting: { -#if SB_HAS(PLAYER_WITH_URL) - if (is_url_based_) { - duration_ = player_bridge_->GetDuration(); - start_date_ = player_bridge_->GetStartDate(); - buffering_state_cb_.Run(kHaveMetadata); - int frame_width; - int frame_height; - player_bridge_->GetVideoResolution(&frame_width, &frame_height); - bool natural_size_changed = (frame_width != natural_size_.width() || - frame_height != natural_size_.height()); - natural_size_ = gfx::Size(frame_width, frame_height); - if (natural_size_changed) { - content_size_change_cb_.Run(); - } - } -#endif // SB_HAS(PLAYER_WITH_URL) - buffering_state_cb_.Run(kPrerollCompleted); - if (!seek_cb_.is_null()) { - CallSeekCB(::media::PIPELINE_OK, ""); - } - if (video_stream_) { - playback_statistics_.OnPresenting( - video_stream_->video_decoder_config()); - } - -#if SB_HAS(PLAYER_WITH_URL) - // Url based player does not support |audio_write_duration_for_preroll_|. - if (is_url_based_) { - break; - } -#endif // SB_HAS(PLAYER_WITH_URL) - - audio_write_duration_for_preroll_ = audio_write_duration_ = + case kSbPlayerStatePresenting: + client_->OnBufferingStateChange(BUFFERING_HAVE_ENOUGH, + BUFFERING_CHANGE_REASON_UNKNOWN); + audio_write_duration_ = HasRemoteAudioOutputs(player_bridge_->GetAudioConfigurations()) ? audio_write_duration_remote_ : audio_write_duration_local_; LOG(INFO) << "SbPlayerBridge reaches kSbPlayerStatePresenting, with audio" << " write duration at " << audio_write_duration_; break; - } case kSbPlayerStateEndOfStream: - ended_cb_.Run(::media::PIPELINE_OK); - ended_ = true; + LOG(INFO) << "StarboardRenderer::OnPlayerStatus() called with " + "kSbPlayerStateEndOfStream."; + client_->OnEnded(); break; case kSbPlayerStateDestroyed: + LOG(INFO) << "StarboardRenderer::OnPlayerStatus() called with " + "kSbPlayerStateDestroyed."; break; } } -void SbPlayerPipeline::OnPlayerError(SbPlayerError error, - const std::string& message) { - DCHECK(task_runner_->RunsTasksInCurrentSequence()); - - // In case if Stop() has been called. - if (!player_bridge_) { - return; - } - -#if SB_HAS(PLAYER_WITH_URL) - if (error >= kSbPlayerErrorMax) { - DCHECK(is_url_based_); - switch (static_cast(error)) { - case kSbUrlPlayerErrorNetwork: - CallErrorCB(::media::PIPELINE_ERROR_NETWORK, message); - break; - case kSbUrlPlayerErrorSrcNotSupported: - CallErrorCB(::media::DEMUXER_ERROR_COULD_NOT_OPEN, message); - - break; - } - return; - } -#endif // SB_HAS(PLAYER_WITH_URL) - switch (error) { - case kSbPlayerErrorDecode: - CallErrorCB(::media::PIPELINE_ERROR_DECODE, message); - break; - case kSbPlayerErrorCapabilityChanged: - CallErrorCB(::media::PIPELINE_ERROR_DECODE, - message.empty() - ? kSbPlayerCapabilityChangedErrorMessage - : ::starboard::FormatString( - "%s: %s", kSbPlayerCapabilityChangedErrorMessage, - message.c_str())); - break; - case kSbPlayerErrorMax: - NOTREACHED(); - break; - } -} - -void SbPlayerPipeline::DelayedNeedData(int max_number_of_buffers_to_write) { - DCHECK(task_runner_->RunsTasksInCurrentSequence()); - if (audio_read_delayed_) { - OnNeedData(DemuxerStream::AUDIO, max_number_of_buffers_to_write); - } -} - -void SbPlayerPipeline::UpdateDecoderConfig(DemuxerStream* stream) { - DCHECK(task_runner_->RunsTasksInCurrentSequence()); - - if (!player_bridge_) { - return; - } - - if (stream->type() == DemuxerStream::AUDIO) { - const AudioDecoderConfig& decoder_config = stream->audio_decoder_config(); - media_metrics_provider_->SetHasAudio(decoder_config.codec()); - player_bridge_->UpdateAudioConfig(decoder_config, stream->mime_type()); - } else { - DCHECK_EQ(stream->type(), DemuxerStream::VIDEO); - const VideoDecoderConfig& decoder_config = stream->video_decoder_config(); - media_metrics_provider_->SetHasVideo(decoder_config.codec()); - base::AutoLock auto_lock(lock_); - bool natural_size_changed = - (decoder_config.natural_size().width() != natural_size_.width() || - decoder_config.natural_size().height() != natural_size_.height()); - natural_size_ = decoder_config.natural_size(); - player_bridge_->UpdateVideoConfig(decoder_config, stream->mime_type()); - if (natural_size_changed) { - content_size_change_cb_.Run(); - } - - playback_statistics_.UpdateVideoConfig(stream->video_decoder_config()); - } -} - -void SbPlayerPipeline::CallSeekCB(PipelineStatus status, - const std::string& error_message) { - if (status == ::media::PIPELINE_OK) { - DCHECK(error_message.empty()); - } - - SeekCB seek_cb; - bool is_initial_preroll; - { - base::AutoLock auto_lock(lock_); - DCHECK(!seek_cb_.is_null()); - seek_cb = std::move(seek_cb_); - is_initial_preroll = is_initial_preroll_; - is_initial_preroll_ = false; - } - seek_cb.Run(status, is_initial_preroll, - AppendStatisticsString(error_message)); -} - -void SbPlayerPipeline::CallErrorCB(PipelineStatus status, - const std::string& error_message) { - DCHECK_NE(status, ::media::PIPELINE_OK); - // Only to record the first error. - if (error_cb_.is_null()) { - return; - } - playback_statistics_.OnError(status, error_message); - ResetAndRunIfNotNull(&error_cb_, status, - AppendStatisticsString(error_message)); -} - -void SbPlayerPipeline::SuspendTask(base::WaitableEvent* done_event) { - DCHECK(task_runner_->RunsTasksInCurrentSequence()); - DCHECK(done_event); - DCHECK(!suspended_); - - if (suspended_) { - done_event->Signal(); - return; - } - - if (player_bridge_) { - // Cancel pending delayed calls to OnNeedData. After - // player_bridge_->Resume(), |player_bridge_| will call OnNeedData again. - audio_read_delayed_ = false; - player_bridge_->Suspend(); - } - - suspended_ = true; - - done_event->Signal(); -} - -void SbPlayerPipeline::ResumeTask(PipelineWindow window, - base::WaitableEvent* done_event) { - DCHECK(task_runner_->RunsTasksInCurrentSequence()); - DCHECK(done_event); - DCHECK(suspended_); - - if (!suspended_) { - last_resume_time_ = Time::Now(); - done_event->Signal(); - return; - } - - window_ = window; - - bool resumable = true; - bool resume_to_background_mode = !SbWindowIsValid(window_); - bool is_audioless = !HasAudio(); - if (resume_to_background_mode && is_audioless) { - // Avoid resuming an audioless video to background mode. SbPlayerBridge will - // try to create an SbPlayer with only the video stream disabled, and may - // crash in this case as SbPlayerCreate() will fail without an audio or - // video stream. - resumable = false; - } - if (player_bridge_ && resumable) { - player_bridge_->Resume(window); - if (!player_bridge_->IsValid()) { - std::string error_message; - { - base::AutoLock auto_lock(lock_); - error_message = player_bridge_->GetPlayerCreationErrorMessage(); - player_bridge_.reset(); - } - std::string time_information = GetTimeInformation(); - LOG(INFO) << "SbPlayerPipeline::ResumeTask failed to create a valid " - "SbPlayerBridge - " - << time_information << " \'" << error_message << "\'"; - CallErrorCB(::media::DECODER_ERROR_NOT_SUPPORTED, - "SbPlayerPipeline::ResumeTask failed to create a valid " - "SbPlayerBridge - " + - time_information + " \'" + error_message + "\'"); - } - } - - suspended_ = false; - last_resume_time_ = Time::Now(); - - done_event->Signal(); -} - -std::string SbPlayerPipeline::AppendStatisticsString( - const std::string& message) const { - DCHECK(task_runner_->RunsTasksInCurrentSequence()); - - if (nullptr == video_stream_) { - return message + ", playback statistics: n/a."; - } else { - return message + ", playback statistics: " + - playback_statistics_.GetStatistics( - video_stream_->video_decoder_config()) + - '.'; - } -} - -std::string SbPlayerPipeline::GetTimeInformation() const { - auto round_time_in_seconds = [](const TimeDelta time) { - const int64_t seconds = time.InSeconds(); - if (seconds < 15) { - return seconds / 5 * 5; - } - if (seconds < 60) { - return seconds / 15 * 15; - } - if (seconds < 3600) { - return std::max(static_cast(60), seconds / 600 * 600); - } - return std::max(static_cast(3600), seconds / 18000 * 18000); - }; - std::string time_since_start = - std::to_string(round_time_in_seconds(base::StartupTimer::TimeElapsed())) + - "s"; - std::string time_since_resume = !last_resume_time_.is_null() - ? std::to_string(round_time_in_seconds( - Time::Now() - last_resume_time_)) + - "s" - : "null"; - return "time since app start: " + time_since_start + - ", time since last resume: " + time_since_resume; -} - -void SbPlayerPipeline::RunSetDrmSystemReadyCB( - DrmSystemReadyCB drm_system_ready_cb) { - TRACE_EVENT0("cobalt::media", "SbPlayerPipeline::RunSetDrmSystemReadyCB"); - set_drm_system_ready_cb_time_ = Time::Now(); - set_drm_system_ready_cb_.Run(drm_system_ready_cb); -} - -void SbPlayerPipeline::SetReadInProgress(DemuxerStream::Type type, - bool in_progress) { - if (type == DemuxerStream::AUDIO) - audio_read_in_progress_ = in_progress; - else - video_read_in_progress_ = in_progress; -} - -bool SbPlayerPipeline::GetReadInProgress(DemuxerStream::Type type) const { - if (type == DemuxerStream::AUDIO) return audio_read_in_progress_; - return video_read_in_progress_; +void StarboardRenderer::OnPlayerError(SbPlayerError error, + const std::string& message) { + // TODO(b/375271948): Implement and verify error reporting. + NOTIMPLEMENTED(); } } // namespace media -} // namespace cobalt diff --git a/media/starboard/starboard_renderer.h b/media/starboard/starboard_renderer.h index ad29852279ee..ac964723b28d 100644 --- a/media/starboard/starboard_renderer.h +++ b/media/starboard/starboard_renderer.h @@ -1,4 +1,4 @@ -// Copyright 2023 The Cobalt Authors. All Rights Reserved. +// Copyright 2024 The Cobalt Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,364 +12,139 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef COBALT_MEDIA_BASE_SBPLAYER_PIPELINE_H_ -#define COBALT_MEDIA_BASE_SBPLAYER_PIPELINE_H_ +#ifndef MEDIA_STARBOARD_STARBOARD_RENDERER_H_ +#define MEDIA_STARBOARD_STARBOARD_RENDERER_H_ -#include -#include #include -#include "base/functional/callback_helpers.h" -#include "base/optional.h" +#include "base/memory/raw_ptr.h" +#include "base/memory/scoped_refptr.h" +#include "base/memory/weak_ptr.h" #include "base/synchronization/lock.h" -#include "base/synchronization/waitable_event.h" -#include "base/task/task_runner.h" +#include "base/task/sequenced_task_runner.h" #include "base/time/time.h" -#include "cobalt/base/c_val.h" -#include "cobalt/math/size.h" -#include "cobalt/media/base/media_export.h" -#include "cobalt/media/base/metrics_provider.h" -#include "cobalt/media/base/pipeline.h" -#include "cobalt/media/base/playback_statistics.h" -#include "cobalt/media/base/sbplayer_bridge.h" -#include "cobalt/media/base/sbplayer_set_bounds_helper.h" -#include "media/base/audio_decoder_config.h" +#include "media/base/cdm_context.h" #include "media/base/decoder_buffer.h" -#include "media/base/demuxer.h" #include "media/base/demuxer_stream.h" -#include "media/base/media_log.h" +#include "media/base/media_resource.h" #include "media/base/pipeline_status.h" -#include "media/base/ranges.h" -#include "media/base/video_decoder_config.h" -#include "starboard/configuration_constants.h" -#include "ui/gfx/geometry/rect.h" -#include "ui/gfx/geometry/size.h" +#include "media/base/renderer.h" +#include "media/base/renderer_client.h" +#include "media/starboard/sbplayer_bridge.h" +#include "third_party/abseil-cpp/absl/types/optional.h" -namespace cobalt { namespace media { -using base::Time; -using base::TimeDelta; - -// SbPlayerPipeline is a PipelineBase implementation that uses the SbPlayer -// interface internally. -class MEDIA_EXPORT SbPlayerPipeline : public Pipeline, - public ::media::DemuxerHost, - public SbPlayerBridge::Host { +// SbPlayer based Renderer implementation, the entry point for all video +// playbacks on Starboard platforms. +class MEDIA_EXPORT StarboardRenderer final : public Renderer, + private SbPlayerBridge::Host { public: - // Constructs a media pipeline that will execute on |task_runner|. - SbPlayerPipeline(SbPlayerInterface* interface, PipelineWindow window, - const scoped_refptr& task_runner, - const GetDecodeTargetGraphicsContextProviderFunc& - get_decode_target_graphics_context_provider_func, - bool allow_resume_after_suspend, - int max_audio_samples_per_write, - bool force_punch_out_by_default, - TimeDelta audio_write_duration_local, - TimeDelta audio_write_duration_remote, MediaLog* media_log, - MediaMetricsProvider* media_metrics_provider, - DecodeTargetProvider* decode_target_provider); - ~SbPlayerPipeline() override; - - void Suspend() override; - // TODO: This is temporary for supporting background media playback. - // Need to be removed with media refactor. - void Resume(PipelineWindow window) override; - - void Start(Demuxer* demuxer, - const SetDrmSystemReadyCB& set_drm_system_ready_cb, - const PipelineStatusCB& ended_cb, const ErrorCB& error_cb, - const SeekCB& seek_cb, const BufferingStateCB& buffering_state_cb, - const base::Closure& duration_change_cb, - const base::Closure& output_mode_change_cb, - const base::Closure& content_size_change_cb, - const std::string& max_video_capabilities, - const int max_video_input_size) override; -#if SB_HAS(PLAYER_WITH_URL) - void Start(const SetDrmSystemReadyCB& set_drm_system_ready_cb, - const OnEncryptedMediaInitDataEncounteredCB& - encrypted_media_init_data_encountered_cb, - const std::string& source_url, const PipelineStatusCB& ended_cb, - const ErrorCB& error_cb, const SeekCB& seek_cb, - const BufferingStateCB& buffering_state_cb, - const base::Closure& duration_change_cb, - const base::Closure& output_mode_change_cb, - const base::Closure& content_size_change_cb) override; -#endif // SB_HAS(PLAYER_WITH_URL) - - void Stop(const base::Closure& stop_cb) override; - void Seek(TimeDelta time, const SeekCB& seek_cb); - bool HasAudio() const override; - bool HasVideo() const override; - - float GetPlaybackRate() const override; - void SetPlaybackRate(float playback_rate) override; - float GetVolume() const override; - void SetVolume(float volume) override; - - TimeDelta GetMediaTime() override; - ::media::Ranges GetBufferedTimeRanges() override; - TimeDelta GetMediaDuration() const override; -#if SB_HAS(PLAYER_WITH_URL) - TimeDelta GetMediaStartDate() const override; -#endif // SB_HAS(PLAYER_WITH_URL) - void GetNaturalVideoSize(gfx::Size* out_size) const override; - std::vector GetAudioConnectors() const override; - - bool DidLoadingProgress() const override; - PipelineStatistics GetStatistics() const override; - SetBoundsCB GetSetBoundsCB() override; - void SetPreferredOutputModeToDecodeToTexture() override; + explicit StarboardRenderer( + const scoped_refptr& task_runner); + + ~StarboardRenderer() final; + + // Renderer implementation. + void Initialize(MediaResource* media_resource, + RendererClient* client, + PipelineStatusCallback init_cb) final; + void SetCdm(CdmContext* cdm_context, CdmAttachedCB cdm_attached_cb) final { + // TODO(b/328305808): Implement encrypted playback. + NOTIMPLEMENTED(); + } + void SetLatencyHint(absl::optional latency_hint) final { + // TODO(b/375271848): Address NOTIMPLEMENTED(). + NOTIMPLEMENTED(); + } + void SetPreservesPitch(bool preserves_pitch) final { + // TODO(b/375271848): Address NOTIMPLEMENTED(). + NOTIMPLEMENTED(); + } + void SetWasPlayedWithUserActivation( + bool was_played_with_user_activation) final { + // TODO(b/375271848): Address NOTIMPLEMENTED(). + NOTIMPLEMENTED(); + } + void Flush(base::OnceClosure flush_cb) final; + void StartPlayingFrom(base::TimeDelta time) final; + void SetPlaybackRate(double playback_rate) final; + void SetVolume(float volume) final; + base::TimeDelta GetMediaTime() final; + void OnSelectedVideoTracksChanged( + const std::vector& enabled_tracks, + base::OnceClosure change_completed_cb) final { + // TODO(b/375271848): Address NOTIMPLEMENTED(). + NOTIMPLEMENTED(); + } + void OnEnabledAudioTracksChanged( + const std::vector& enabled_tracks, + base::OnceClosure change_completed_cb) final { + // TODO(b/375271848): Address NOTIMPLEMENTED(). + NOTIMPLEMENTED(); + } + RendererType GetRendererType() final { + // Reuse `kRendererImpl` to avoid introducing a new renderer type. + // TODO(b/375278384): Properly setup the renderer type. + return RendererType::kRendererImpl; + } private: - // Used to post parameters to SbPlayerPipeline::StartTask() as the number of - // parameters exceed what base::Bind() can support. - struct StartTaskParameters { - ::media::Demuxer* demuxer; - SetDrmSystemReadyCB set_drm_system_ready_cb; - ::media::PipelineStatusCB ended_cb; - ErrorCB error_cb; - Pipeline::SeekCB seek_cb; - Pipeline::BufferingStateCB buffering_state_cb; - base::Closure duration_change_cb; - base::Closure output_mode_change_cb; - base::Closure content_size_change_cb; - std::string max_video_capabilities; - int max_video_input_size; -#if SB_HAS(PLAYER_WITH_URL) - std::string source_url; - bool is_url_based; -#endif // SB_HAS(PLAYER_WITH_URL) - }; - - void StartTask(StartTaskParameters parameters); - void SetVolumeTask(float volume); - void SetPlaybackRateTask(float volume); - void SetDurationTask(TimeDelta duration); - - // DemuxerHost implementation. - void OnBufferedTimeRangesChanged( - const ::media::Ranges& ranges) override; - void SetDuration(TimeDelta duration) override; - void OnDemuxerError(PipelineStatus error) override; - -#if SB_HAS(PLAYER_WITH_URL) - void CreateUrlPlayer(const std::string& source_url); - void SetDrmSystem(SbDrmSystem drm_system); -#endif // SB_HAS(PLAYER_WITH_URL) - void CreatePlayer(SbDrmSystem drm_system); - - void OnDemuxerInitialized(PipelineStatus status); - void OnDemuxerSeeked(PipelineStatus status); - void OnDemuxerStopped(); - void OnDemuxerStreamRead( - ::media::DemuxerStream::Type type, int max_number_buffers_to_read, - ::media::DemuxerStream::Status status, - const std::vector> buffers); - // SbPlayerBridge::Host implementation. - void OnNeedData(::media::DemuxerStream::Type type, - int max_number_of_buffers_to_write) override; - void OnPlayerStatus(SbPlayerState state) override; - void OnPlayerError(SbPlayerError error, const std::string& message) override; - - // Used to make a delayed call to OnNeedData() if |audio_read_delayed_| is - // true. If |audio_read_delayed_| is false, that means the delayed call has - // been cancelled due to a seek. - void DelayedNeedData(int max_number_of_buffers_to_write); + void CreatePlayerBridge(PipelineStatusCallback init_cb); + void UpdateDecoderConfig(DemuxerStream* stream); + void OnDemuxerStreamRead(DemuxerStream* stream, + DemuxerStream::Status status, + DemuxerStream::DecoderBufferVector buffers); - void UpdateDecoderConfig(::media::DemuxerStream* stream); - void CallSeekCB(PipelineStatus status, const std::string& error_message); - void CallErrorCB(PipelineStatus status, const std::string& error_message); + void OnNeedData(DemuxerStream::Type type, + int max_number_of_buffers_to_write) final; + void OnPlayerStatus(SbPlayerState state) final; + void OnPlayerError(SbPlayerError error, const std::string& message) final; - void SuspendTask(base::WaitableEvent* done_event); - void ResumeTask(PipelineWindow window, base::WaitableEvent* done_event); - - // Store the media time retrieved by GetMediaTime so we can cache it as an - // estimate and avoid calling SbPlayerGetInfo too frequently. - void StoreMediaTime(TimeDelta media_time); - - // Retrieve the statistics as a string and append to message. - std::string AppendStatisticsString(const std::string& message) const; - - // Get information on the time since app start and the time since the last - // playback resume. - std::string GetTimeInformation() const; - - void RunSetDrmSystemReadyCB(DrmSystemReadyCB drm_system_ready_cb); - - void SetReadInProgress(::media::DemuxerStream::Type type, bool in_progress); - bool GetReadInProgress(::media::DemuxerStream::Type type) const; - - int GetDefaultMaxBuffers(AudioCodec codec, TimeDelta duration_to_write, - bool is_preroll); - int GetEstimatedMaxBuffers(TimeDelta write_duration, - TimeDelta time_ahead_of_playback, bool is_preroll); - - // An identifier string for the pipeline, used in CVal to identify multiple - // pipelines. - const std::string pipeline_identifier_; - - // A wrapped interface of starboard player functions, which will be used in - // underlying SbPlayerBridge. - SbPlayerInterface* sbplayer_interface_; - - // Message loop used to execute pipeline tasks. It is thread-safe. scoped_refptr task_runner_; - // Whether we should save DecoderBuffers for resume after suspend. - const bool allow_resume_after_suspend_; - - // The number of audio samples per write that is allowed at once. - // The minimum number of |max_audio_samples_per_write_| and - // SbPlayerGetMaximumNumberOfSamplesPerWrite() specifies - // the maximum number of samples SbPlayerWriteSamples() can accept. - const int max_audio_samples_per_write_; - - // The default output mode passed to `SbPlayerGetPreferredOutputMode()`. - SbPlayerOutputMode default_output_mode_ = kSbPlayerOutputModeInvalid; - - // The window this player associates with. It should only be assigned in the - // dtor and accessed once by SbPlayerCreate(). - PipelineWindow window_; - - // Call to get the SbDecodeTargetGraphicsContextProvider for SbPlayerCreate(). - const GetDecodeTargetGraphicsContextProviderFunc - get_decode_target_graphics_context_provider_func_; - - // Lock used to serialize access for the following member variables. - mutable base::Lock lock_; - - // Amount of available buffered data. Set by filters. - ::media::Ranges buffered_time_ranges_; - - // True when AddBufferedByteRange() has been called more recently than - // DidLoadingProgress(). - mutable bool did_loading_progress_; - - // Video's natural width and height. Set by filters. - gfx::Size natural_size_; - - // Current volume level (from 0.0f to 1.0f). This value is set immediately - // via SetVolume() and a task is dispatched on the task runner to notify the - // filters. - base::CVal volume_; - - // Current playback rate (>= 0.0f). This value is set immediately via - // SetPlaybackRate() and a task is dispatched on the task runner to notify - // the filters. - base::CVal playback_rate_; - - // The saved audio and video demuxer streams. Note that it is safe to store - // raw pointers of the demuxer streams, as the Demuxer guarantees that its - // |DemuxerStream|s live as long as the Demuxer itself. - ::media::DemuxerStream* audio_stream_ = nullptr; - ::media::DemuxerStream* video_stream_ = nullptr; - - mutable PipelineStatistics statistics_; + raw_ptr audio_stream_ = nullptr; + raw_ptr video_stream_ = nullptr; + // TODO(b/375273774): Consider calling `void OnWaiting(WaitingReason reason)` + // on `client_`. + // TODO(b/375274109): Investigate whether we should call + // `void OnVideoFrameRateChange(absl::optional fps)` + // on `client_`? + raw_ptr client_ = nullptr; + + DefaultSbPlayerInterface sbplayer_interface_; + // TODO(b/326652276): Support audio write duration. + const base::TimeDelta audio_write_duration_local_ = base::Milliseconds(500); + const base::TimeDelta audio_write_duration_remote_ = base::Seconds(10); + // TODO(b/375674101): Support batched samples write. + const int max_audio_samples_per_write_ = 1; + + base::Lock lock_; + std::unique_ptr player_bridge_; - // The following member variables are only accessed by tasks posted to - // |task_runner_|. + bool player_bridge_initialized_ = false; + std::optional playing_start_from_time_; - // Temporary callback used for Stop(). - base::Closure stop_cb_; + base::OnceClosure pending_flush_cb_; - // Permanent callbacks passed in via Start(). - SetDrmSystemReadyCB set_drm_system_ready_cb_; - PipelineStatusCB ended_cb_; - ErrorCB error_cb_; - BufferingStateCB buffering_state_cb_; - base::Closure duration_change_cb_; - base::Closure output_mode_change_cb_; - base::Closure content_size_change_cb_; - base::Optional decode_to_texture_output_mode_; -#if SB_HAS(PLAYER_WITH_URL) - SbPlayerBridge::OnEncryptedMediaInitDataEncounteredCB - on_encrypted_media_init_data_encountered_cb_; -#endif // SB_HAS(PLAYER_WITH_URL) + base::TimeDelta audio_write_duration_ = audio_write_duration_local_; - // Demuxer reference used for setting the preload value. - Demuxer* demuxer_ = nullptr; bool audio_read_in_progress_ = false; - bool audio_read_delayed_ = false; bool video_read_in_progress_ = false; - base::CVal duration_; - -#if SB_HAS(PLAYER_WITH_URL) - TimeDelta start_date_; - bool is_url_based_; -#endif // SB_HAS(PLAYER_WITH_URL) - - scoped_refptr set_bounds_helper_; - - // The following member variables can be accessed from WMPI thread but all - // modifications to them happens on the pipeline thread. So any access of - // them from the WMPI thread and any modification to them on the pipeline - // thread has to guarded by lock. Access to them from the pipeline thread - // needn't to be guarded. - - // Temporary callback used for Start() and Seek(). - SeekCB seek_cb_; - TimeDelta seek_time_; - std::unique_ptr player_bridge_; - bool is_initial_preroll_ = true; - base::CVal started_; - base::CVal suspended_; - base::CVal stopped_; - base::CVal ended_; - base::CVal player_state_; - - MediaMetricsProvider* media_metrics_provider_; - - DecodeTargetProvider* decode_target_provider_; - - const TimeDelta audio_write_duration_local_; - const TimeDelta audio_write_duration_remote_; - - // The two variables below should always contain the same value. They are - // kept as separate variables so we can keep the existing implementation as - // is, which simplifies the implementation across multiple Starboard versions. - TimeDelta audio_write_duration_; - TimeDelta audio_write_duration_for_preroll_ = audio_write_duration_; - - // Only call GetMediaTime() from OnNeedData if it has been - // |kMediaTimeCheckInterval| since the last call to GetMediaTime(). - static constexpr TimeDelta kMediaTimeCheckInterval = - TimeDelta::FromMicroseconds(100); - // Timestamp for the last written audio. - TimeDelta timestamp_of_last_written_audio_; - // Indicates if video end of stream has been written into the underlying - // player. - bool is_video_eos_written_ = false; - TimeDelta last_audio_sample_interval_ = TimeDelta::FromMicroseconds(0); - int last_estimated_max_buffers_for_preroll_ = 1; - - // Last media time reported by GetMediaTime(). - base::CVal last_media_time_; - // Timestamp microseconds when we last checked the media time. - Time last_time_media_time_retrieved_; - // Counter for retrograde media time. - size_t retrograde_media_time_counter_ = 0; - // The maximum video playback capabilities required for the playback. - base::CVal max_video_capabilities_; - // Set the maximum size in bytes of an input buffer for video. - int max_video_input_size_; - - PlaybackStatistics playback_statistics_; - Time last_resume_time_; + double playback_rate_ = 0.; + float volume_ = 1.0f; - Time set_drm_system_ready_cb_time_; + uint32_t last_video_frames_decoded_ = 0; + uint32_t last_video_frames_dropped_ = 0; - // Message to signal a capability changed error. - // "MEDIA_ERR_CAPABILITY_CHANGED" must be in the error message to be - // understood as a capability changed error. Do not change this message. - static inline constexpr const char* kSbPlayerCapabilityChangedErrorMessage = - "MEDIA_ERR_CAPABILITY_CHANGED"; + base::WeakPtrFactory weak_factory_{this}; + base::WeakPtr weak_this_{weak_factory_.GetWeakPtr()}; - DISALLOW_COPY_AND_ASSIGN(SbPlayerPipeline); + StarboardRenderer(const StarboardRenderer&) = delete; + StarboardRenderer& operator=(const StarboardRenderer&) = delete; }; } // namespace media -} // namespace cobalt -#endif // COBALT_MEDIA_BASE_SBPLAYER_PIPELINE_H_ +#endif // MEDIA_STARBOARD_STARBOARD_RENDERER_H_ diff --git a/media/starboard/starboard_utils.cc b/media/starboard/starboard_utils.cc index 25871101dedc..5dceed49ea57 100644 --- a/media/starboard/starboard_utils.cc +++ b/media/starboard/starboard_utils.cc @@ -12,11 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#if !defined(STARBOARD) -#error "This file only works with Cobalt/Starboard." -#endif // !defined(STARBOARD) - -#include "media/base/starboard_utils.h" +#include "media/starboard/starboard_utils.h" #include @@ -74,15 +70,15 @@ SbMediaAudioCodec MediaAudioCodecToSbMediaAudioCodec(AudioCodec codec) { return kSbMediaAudioCodecFlac; case AudioCodec::kPCM: return kSbMediaAudioCodecPcm; +#if COBALT_MEDIA_ENABLE_IAMF_SUPPORT case AudioCodec::kIAMF: return kSbMediaAudioCodecIamf; +#endif // COBALT_MEDIA_ENABLE_IAMF_SUPPORT default: // Cobalt only supports a subset of audio codecs defined by Chromium. - DLOG(ERROR) << "Unsupported audio codec " << GetCodecName(codec); + LOG(ERROR) << "Unsupported audio codec " << GetCodecName(codec); return kSbMediaAudioCodecNone; } - NOTREACHED(); - return kSbMediaAudioCodecNone; } SbMediaVideoCodec MediaVideoCodecToSbMediaVideoCodec(VideoCodec codec) { @@ -105,11 +101,9 @@ SbMediaVideoCodec MediaVideoCodecToSbMediaVideoCodec(VideoCodec codec) { return kSbMediaVideoCodecAv1; default: // Cobalt only supports a subset of video codecs defined by Chromium. - DLOG(ERROR) << "Unsupported video codec " << GetCodecName(codec); + LOG(ERROR) << "Unsupported video codec " << GetCodecName(codec); return kSbMediaVideoCodecNone; } - NOTREACHED(); - return kSbMediaVideoCodecNone; } SbMediaAudioStreamInfo MediaAudioConfigToSbMediaAudioStreamInfo( @@ -126,7 +120,8 @@ SbMediaAudioStreamInfo MediaAudioConfigToSbMediaAudioStreamInfo( ChannelLayoutToChannelCount(audio_decoder_config.channel_layout()); audio_stream_info.samples_per_second = audio_decoder_config.samples_per_second(); - audio_stream_info.bits_per_sample = audio_decoder_config.bits_per_channel(); + audio_stream_info.bits_per_sample = + audio_decoder_config.bytes_per_channel() * 8; const auto& extra_data = audio_stream_info.codec == kSbMediaAudioCodecAac ? audio_decoder_config.aac_extra_data() @@ -201,8 +196,8 @@ void FillDrmSampleInfo(const scoped_refptr& buffer, drm_info->subsample_count = config->subsamples().size(); if (drm_info->subsample_count > 0) { - COMPILE_ASSERT(sizeof(SbDrmSubSampleMapping) == sizeof(SubsampleEntry), - SubSampleEntrySizesMatch); + static_assert(sizeof(SbDrmSubSampleMapping) == sizeof(SubsampleEntry), + "SubSampleEntrySizesMatch"); drm_info->subsample_mapping = reinterpret_cast( &config->subsamples()[0]); @@ -223,8 +218,10 @@ void FillDrmSampleInfo(const scoped_refptr& buffer, // Ensure that the enums in starboard/media.h match enums in // VideoColorSpace and gfx::ColorSpace. -#define ENUM_EQ(a, b) \ - COMPILE_ASSERT(static_cast(a) == static_cast(b), mismatching_enums) +#define ENUM_EQ(a, b) \ + static_assert(static_cast(a) == static_cast(b), \ + "mismatching_" \ + "enums") // Ensure PrimaryId enums convert correctly. ENUM_EQ(kSbMediaPrimaryIdReserved0, VideoColorSpace::PrimaryID::INVALID); @@ -366,6 +363,7 @@ SbMediaColorMetadata MediaToSbMediaColorMetadata( } int GetSbMediaVideoBufferBudget(const VideoDecoderConfig* video_config, const std::string& mime_type) { +#if COBALT_MEDIA_ENABLE_DECODE_BUFFER_BUDGET if (!video_config) { return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget( kSbMediaVideoCodecH264, 1920, 1080, 8); @@ -377,6 +375,9 @@ int GetSbMediaVideoBufferBudget(const VideoDecoderConfig* video_config, auto codec = MediaVideoCodecToSbMediaVideoCodec(video_config->codec()); return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget( codec, width, height, bits_per_pixel); +#else // COBALT_MEDIA_ENABLE_DECODE_BUFFER_BUDGET + return 100 * 1024 * 1024; +#endif // COBALT_MEDIA_ENABLE_DECODE_BUFFER_BUDGET } std::string ExtractCodecs(const std::string& mime_type) { diff --git a/media/starboard/starboard_utils.h b/media/starboard/starboard_utils.h index 87119183ce0c..8f2a174efc03 100644 --- a/media/starboard/starboard_utils.h +++ b/media/starboard/starboard_utils.h @@ -12,12 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef MEDIA_BASE_STARBOARD_UTILS_H_ -#define MEDIA_BASE_STARBOARD_UTILS_H_ - -#if !defined(STARBOARD) -#error "This file only works with Cobalt/Starboard." -#endif // !defined(STARBOARD) +#ifndef MEDIA_STARBOARD_STARBOARD_UTILS_H_ +#define MEDIA_STARBOARD_STARBOARD_UTILS_H_ #include "media/base/audio_codecs.h" #include "media/base/audio_decoder_config.h" @@ -65,4 +61,4 @@ std::string ExtractCodecs(const std::string& mime_type); } // namespace media -#endif // MEDIA_BASE_STARBOARD_UTILS_H_ +#endif // MEDIA_STARBOARD_STARBOARD_UTILS_H_ diff --git a/media/starboard/starboard_utils_test.cc b/media/starboard/starboard_utils_test.cc index a2feb6d2e101..88368892a5cc 100644 --- a/media/starboard/starboard_utils_test.cc +++ b/media/starboard/starboard_utils_test.cc @@ -16,7 +16,7 @@ #error "This file only works with Cobalt/Starboard." #endif // !defined(STARBOARD) -#include "media/base/starboard_utils.h" +#include "media/starboard/starboard_utils.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/starboard/android/shared/main.cc b/starboard/android/shared/main.cc index 175e92acc141..bf0f031496e7 100644 --- a/starboard/android/shared/main.cc +++ b/starboard/android/shared/main.cc @@ -15,9 +15,12 @@ #include "starboard/common/log.h" #include "starboard/export.h" +// TODO(b/375459298); Investigate whether we should enable main() on Android TV +#if 0 extern "C" SB_EXPORT_PLATFORM int main(int argc, char** argv) { // main() is never called on Android. However, the cobalt_bin // target requires it to be there. SB_NOTREACHED(); return 0; } +#endif // 0 diff --git a/third_party/blink/renderer/modules/mediasource/BUILD.gn b/third_party/blink/renderer/modules/mediasource/BUILD.gn index 513d0e9d92b7..1eb0a04977eb 100644 --- a/third_party/blink/renderer/modules/mediasource/BUILD.gn +++ b/third_party/blink/renderer/modules/mediasource/BUILD.gn @@ -52,4 +52,7 @@ blink_modules_sources("mediasource") { # Ensure the generated webcodecs v8 config and chunk bindings are available. "//third_party/blink/renderer/modules/webcodecs:webcodecs", ] + if (is_cobalt) { + deps += [ "//starboard/build:starboard_buildflags" ] + } } diff --git a/third_party/blink/renderer/modules/mediasource/media_source.cc b/third_party/blink/renderer/modules/mediasource/media_source.cc index bffaac9e4631..4989c73e8b6c 100644 --- a/third_party/blink/renderer/modules/mediasource/media_source.cc +++ b/third_party/blink/renderer/modules/mediasource/media_source.cc @@ -57,6 +57,12 @@ #include "third_party/blink/renderer/platform/wtf/functional.h" #include "third_party/blink/renderer/platform/wtf/text/wtf_string.h" +// For BUILDFLAG(USE_STARBOARD_MEDIA) +#include "build/buildflag.h" +#if BUILDFLAG(IS_COBALT) +#include "starboard/build/starboard_buildflags.h" +#endif // BUILDFLAG(IS_COBALT) + using blink::WebMediaSource; using blink::WebSourceBuffer; @@ -492,6 +498,12 @@ bool MediaSource::isTypeSupported(ExecutionContext* context, bool result = IsTypeSupportedInternal( context, type, true /* Require fully specified mime and codecs */); DVLOG(2) << __func__ << "(" << type << ") -> " << (result ? "true" : "false"); +#if BUILDFLAG(IS_COBALT) +#if BUILDFLAG(USE_STARBOARD_MEDIA) + LOG(INFO) << __func__ << "(" << type << ") -> " + << (result ? "true" : "false"); +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) +#endif // BUILDFLAG(IS_COBALT) return result; } @@ -499,6 +511,35 @@ bool MediaSource::isTypeSupported(ExecutionContext* context, bool MediaSource::IsTypeSupportedInternal(ExecutionContext* context, const String& type, bool enforce_codec_specificity) { +#if BUILDFLAG(USE_STARBOARD_MEDIA) + // TODO(b/322021829): This is a workaround to claim 4K playback support. It + // should be replaced by a proper implementation of + // MediaSource.isTypeSupported(). + if (type.Find("99") != kNotFound || type.Find("catavision") != kNotFound || + type.Find("invalidformat") != kNotFound || + type.Find("bitrate=2000000000") != kNotFound || + type.Find("decode-to-texture=nope") != kNotFound || + type.Find("decode-to-texture=true") != kNotFound) { + return false; + } + + // Reject 8k + if (type.Find("7680") != kNotFound) { + return false; + } + + /* + // Reject 4k + if (type.Find("3840") != kNotFound) { + return false; + } + + // Reject 2k + if (type.Find("2560") != kNotFound) { + return false; + }*/ +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) + // Even after ExecutionContext teardown notification, bindings may still call // code-behinds for a short while. If |context| is null, this is likely // happening. To prevent possible null deref of |context| in this path, claim diff --git a/third_party/blink/renderer/platform/media/BUILD.gn b/third_party/blink/renderer/platform/media/BUILD.gn index f679e9348ee0..d8e88fcd6b8d 100644 --- a/third_party/blink/renderer/platform/media/BUILD.gn +++ b/third_party/blink/renderer/platform/media/BUILD.gn @@ -104,6 +104,9 @@ component("media") { "//media/remoting:remoting_constants", "//third_party/blink/public/strings:strings_grit", ] + if (is_cobalt) { + deps += [ "//starboard/build:starboard_buildflags" ] + } } if (enable_hls_demuxer) { diff --git a/third_party/blink/renderer/platform/media/web_media_player_impl.cc b/third_party/blink/renderer/platform/media/web_media_player_impl.cc index d0b3f2bf6f23..63b51d6d98f0 100644 --- a/third_party/blink/renderer/platform/media/web_media_player_impl.cc +++ b/third_party/blink/renderer/platform/media/web_media_player_impl.cc @@ -65,6 +65,8 @@ #include "mojo/public/cpp/bindings/pending_remote.h" #include "net/base/data_url.h" #include "services/device/public/mojom/battery_monitor.mojom-blink.h" +// For BUILDFLAG(USE_STARBOARD_MEDIA) +#include "starboard/build/starboard_buildflags.h" #include "third_party/blink/public/common/media/display_type.h" #include "third_party/blink/public/common/media/watch_time_reporter.h" #include "third_party/blink/public/common/thread_safe_browser_interface_broker_proxy.h" @@ -98,6 +100,15 @@ #include "third_party/blink/renderer/platform/media/web_media_source_impl.h" #include "ui/gfx/geometry/size.h" +// For BUILDFLAG(USE_STARBOARD_MEDIA) +#include "build/buildflag.h" +#if BUILDFLAG(IS_COBALT) +#include "starboard/build/starboard_buildflags.h" +#if BUILDFLAG(USE_STARBOARD_MEDIA) +#include "media/starboard/starboard_renderer.h" +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) +#endif // BUILDFLAG(IS_COBALT) + #if BUILDFLAG(ENABLE_HLS_DEMUXER) #include "third_party/blink/renderer/platform/media/hls_data_source_provider_impl.h" #endif // BUILDFLAG(ENABLE_HLS_DEMUXER) @@ -2796,6 +2807,19 @@ std::unique_ptr WebMediaPlayerImpl::CreateRenderer( &WebMediaPlayerImpl::OnOverlayInfoRequested, weak_this_)); #endif +#if BUILDFLAG(IS_COBALT) +#if BUILDFLAG(USE_STARBOARD_MEDIA) + // TODO(b/375278384): Select the StarboardRenderer properly instead of + // hard coding. + + // `media_task_runner_` is always true, use an if statement to avoid + // potential build warning on unreachable code. + if (media_task_runner_) { + return std::make_unique(media_task_runner_); + } +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) +#endif // BUILDFLAG(IS_COBALT) + if (renderer_type) { DVLOG(1) << __func__ << ": renderer_type=" << static_cast(renderer_type.value());