From 871ec472c0b1dda0d5a557f618d1ee0b422a3daf Mon Sep 17 00:00:00 2001 From: Andrew Polk Date: Tue, 30 Jul 2024 16:07:54 -0700 Subject: [PATCH] fix: Play all videos on the page (BL-13705) --- src/bloom-player-core.tsx | 162 +++++++++++++++++---------------- src/bloom-player-ui.less | 2 +- src/dragActivityRuntime.ts | 3 +- src/narration.ts | 15 +++- src/video.ts | 179 ++++++++++++++++++++----------------- 5 files changed, 194 insertions(+), 167 deletions(-) diff --git a/src/bloom-player-core.tsx b/src/bloom-player-core.tsx index cf0b742..72a17b9 100644 --- a/src/bloom-player-core.tsx +++ b/src/bloom-player-core.tsx @@ -1007,77 +1007,79 @@ export class BloomPlayerCore extends React.Component { } private showReplayButton(pageVideoData: IPageVideoComplete | undefined) { - if (!pageVideoData?.video || !pageVideoData!.page) { - return; // paranoia, and allows us to assume they are defined without ! everywhere. - } - const parent = pageVideoData.video.parentElement!; - let replayButton = document.getElementById("replay-button"); - if (!replayButton) { - replayButton = document.createElement("div"); - replayButton.setAttribute("id", "replay-button"); - replayButton.style.position = "absolute"; - replayButton.style.display = "none"; - ReactDOM.render( - { - // in storybook, I was seeing the page jump around as I clicked the button. - // Guessing it was somehow caused by something higher up also responding to - // the click, I put these in to try to stop it, but didn't succeed. - // If we get the behavior in production, we'll need to try some more. - args.preventDefault(); - args.stopPropagation(); - // This not only starts the video, it should put everything in the right - // state, including stopping any audio. If we change our minds about - // always playing video first, or decide to support more than one video - // on a page, we'll need something smarter here. - this.resetForNewPageAndPlay( - BloomPlayerCore.currentPage! - ); - }} - onTouchStart={args => { - // This prevents the toolbar from toggling if we start a touch on the Replay button. - // If the touch ends up being a tap, then onClick will get processed too. - this.setState({ ignorePhonyClick: true }); - }} - onMouseDown={args => { - // another attempt to stop the jumping around. - args.stopPropagation(); - }} - />, - replayButton + pageVideoData?.videos.forEach((video, index) => { + const parent = video.parentElement!; + let replayButton = document.getElementById( + `replay-button-${index}` ); - } - // from https://developer.mozilla.org/en-US/docs/Web/HTTP/Browser_detection_using_the_user_agent - // const UA = navigator.userAgent; - // const isWebkit = - // /\b(iPad|iPhone|iPod)\b/.test(UA) && - // /WebKit/.test(UA) && - // !/Edge/.test(UA) && - // !window.MSStream; - //if (isWebkit || /chrome/.test(UA.toLowerCase())) { - // Due to a bug in Chrome and Safari (and probably other Webkit-based browsers), - // we can't be sure the button will show up if we place it over the video - // So, instead, we hide the video to make room for it. That can seem a very abrupt change, - // so we fade the video out before replacing it with the button. - // We have tried keeping the behavior in the 'else' commented out below, which we would - // prefer, for those browsers which can do it correctly; but it has proved too difficult - // to detect which those are. - parent.insertBefore(replayButton, pageVideoData.video); // there but still display:none - pageVideoData.video.classList.add("fade-out"); - setTimeout(() => { - pageVideoData.video.style.display = "none"; - replayButton!.style.display = "block"; - pageVideoData.video.classList.remove("fade-out"); - }, 1000); - // } else { - // // On correctly-implemented browsers, it's neater to just overlay the button on top of the video. - // // keeping this code in case we decide to try again sometime, and to remember what we'd like to - // // do, but it will likely be a long time before we can be sure no problem browsers are around any more. - // replayButton.style.position = "absolute"; - // parent.appendChild(replayButton); - // replayButton!.style.display = "block"; - // } + if (!replayButton) { + replayButton = document.createElement("div"); + replayButton.setAttribute("id", `replay-button-${index}`); + replayButton.classList.add("replay-button"); + replayButton.style.position = "absolute"; + replayButton.style.display = "none"; + ReactDOM.render( + { + // in storybook, I was seeing the page jump around as I clicked the button. + // Guessing it was somehow caused by something higher up also responding to + // the click, I put these in to try to stop it, but didn't succeed. + // If we get the behavior in production, we'll need to try some more. + args.preventDefault(); + args.stopPropagation(); + + video.style.display = "block"; + if (replayButton) + // replayButton is always defined, but TS doesn't know that. + replayButton.style.display = "none"; + this.video.replaySingleVideo(video); + }} + onTouchStart={args => { + // This prevents the toolbar from toggling if we start a touch on the Replay button. + // If the touch ends up being a tap, then onClick will get processed too. + this.setState({ ignorePhonyClick: true }); + }} + onMouseDown={args => { + // another attempt to stop the jumping around. + args.stopPropagation(); + }} + />, + replayButton + ); + } + + // from https://developer.mozilla.org/en-US/docs/Web/HTTP/Browser_detection_using_the_user_agent + // const UA = navigator.userAgent; + // const isWebkit = + // /\b(iPad|iPhone|iPod)\b/.test(UA) && + // /WebKit/.test(UA) && + // !/Edge/.test(UA) && + // !window.MSStream; + //if (isWebkit || /chrome/.test(UA.toLowerCase())) { + // Due to a bug in Chrome and Safari (and probably other Webkit-based browsers), + // we can't be sure the button will show up if we place it over the video + // So, instead, we hide the video to make room for it. That can seem a very abrupt change, + // so we fade the video out before replacing it with the button. + // We have tried keeping the behavior in the 'else' commented out below, which we would + // prefer, for those browsers which can do it correctly; but it has proved too difficult + // to detect which those are. + parent.insertBefore(replayButton, video); // there but still display:none + video.classList.add("fade-out"); + setTimeout(() => { + video.style.display = "none"; + replayButton!.style.display = "block"; + video.classList.remove("fade-out"); + }, 1000); + // } else { + // // On correctly-implemented browsers, it's neater to just overlay the button on top of the video. + // // keeping this code in case we decide to try again sometime, and to remember what we'd like to + // // do, but it will likely be a long time before we can be sure no problem browsers are around any more. + // replayButton.style.position = "absolute"; + // parent.appendChild(replayButton); + // replayButton!.style.display = "block"; + // } + }); } // We need named functions for each LiteEvent handler, so that we can unsubscribe them @@ -2768,20 +2770,24 @@ export class BloomPlayerCore extends React.Component { } player.sendUpdateOfBookProgressReportToExternalContext(); } + // This should only be called when NOT paused, because it will begin to play audio and highlighting // and animation from the beginning of the page. private resetForNewPageAndPlay(bloomPage: HTMLElement): void { if (this.props.paused) { return; // shouldn't call when paused } - const replayButton = document.getElementById("replay-button"); - if (replayButton) { - replayButton.style.display = "none"; - const video = replayButton.parentElement?.getElementsByTagName( - "video" - )[0]; - if (video) { - video.style.display = ""; + const replayButtons = document.getElementsByClassName("replay-button"); + if (replayButtons) { + for (let i = 0; i < replayButtons.length; i++) { + const replayButton = replayButtons[i] as HTMLElement; + replayButton.style.display = "none"; + const video = replayButton.parentElement?.getElementsByTagName( + "video" + )[0]; + if (video) { + video.style.display = ""; + } } } this.animation.PlayAnimation(); // get rid of classes that made it pause diff --git a/src/bloom-player-ui.less b/src/bloom-player-ui.less index f136565..f01b5da 100644 --- a/src/bloom-player-ui.less +++ b/src/bloom-player-ui.less @@ -247,7 +247,7 @@ We just make them follow the main content normally. */ } } -#replay-button { +.bloomPlayer .replay-button { font-size: 45px; left: calc(50% - 22px); top: calc(50% - 22px); diff --git a/src/dragActivityRuntime.ts b/src/dragActivityRuntime.ts index 5fadf27..f26b212 100644 --- a/src/dragActivityRuntime.ts +++ b/src/dragActivityRuntime.ts @@ -6,12 +6,11 @@ // For now, in Bloom desktop, this file is only used in the Play tab of drag activities, // so both live there. In Bloom player, both live in the root src directory. // In the long run, the answer is probably a folder, or even an npm package, for all -// the stuff that the two progams share...or maybe we can make bloom player publish +// the stuff that the two programs share...or maybe we can make bloom player publish // these files along with the output bundle and have bloom desktop use them from there. // For now, though, it's much easier to just edit them and have them built automatically // than to have this code in another repo. -import { get } from "jquery"; import { kAudioSentence, playAllAudio, diff --git a/src/narration.ts b/src/narration.ts index 03325d2..b2a5f7a 100644 --- a/src/narration.ts +++ b/src/narration.ts @@ -1220,7 +1220,12 @@ export function hidingPage() { // Play the specified elements, one after the other. When the last completes (or at once if the array is empty), // perform the 'then' action (typically used to play narration, which we put after videos). -// Todo: Bloom Player version, at least, should work with play/pause/resume/change page architecture. +// +// Note, there is a very similar function in narration.ts. It would be nice to combine them, but +// there are various reasons that is difficult at the moment. e.g.: +// 1. See comment below about sharing code with Bloom Desktop. +// 2. The other version handles play/pause which doesn't apply in BloomDesktop. +// // (This function would be more natural in video.ts. But at least for now I'm trying to minimize the // number of source files shared with Bloom Desktop, and we need this for Bloom Games.) export function playAllVideo(elements: HTMLVideoElement[], then: () => void) { @@ -1244,5 +1249,11 @@ export function playAllVideo(elements: HTMLVideoElement[], then: () => void) { ); // Review: do we need to do something to let the rest of the world know about this? setCurrentPlaybackMode(PlaybackMode.VideoPlaying); - video.play(); + + const promise = video.play(); + // If there is an error, try to continue with the next video. + promise?.catch(reason => { + console.error("Video play failed", reason); + this.playAllVideo(elements.slice(1), then); + }); } diff --git a/src/video.ts b/src/video.ts index 540e7e4..e894aec 100644 --- a/src/video.ts +++ b/src/video.ts @@ -11,42 +11,41 @@ import { export interface IPageVideoComplete { page: HTMLElement; - video: HTMLElement; + videos: HTMLVideoElement[]; } export class Video { private currentPage: HTMLDivElement; private currentVideoElement: HTMLVideoElement | undefined; + private currentVideoStartTime: number = 0; + private isPlayingSingleVideo: boolean = false; public PageVideoComplete: LiteEvent; public static pageHasVideo(page: HTMLElement): boolean { - return !!page.getElementsByTagName("video").length; + return !!Video.getVideoElements(page).length; } - private videoStartTime: number; - private videoEnded: boolean; - // Work we prefer to do before the page is visible. This makes sure that when the video // is loaded it will begin to play automatically. - // Enhance: someday we may need to handle multiple videos per page? public HandlePageBeforeVisible(page: HTMLElement) { this.currentPage = page as HTMLDivElement; if (!Video.pageHasVideo(this.currentPage)) { this.currentVideoElement = undefined; return; } - this.currentVideoElement = this.getFirstVideo() as HTMLVideoElement; - if (this.currentVideoElement.hasAttribute("controls")) { - this.currentVideoElement.removeAttribute("controls"); - } - if (!this.currentVideoElement.hasAttribute("playsinline")) { - this.currentVideoElement.setAttribute("playsinline", "true"); - } - if (this.currentVideoElement.currentTime !== 0) { - // in case we previously played this video and are returning to this page... - this.currentVideoElement.currentTime = 0; - } + this.getVideoElements().forEach(videoElement => { + if (videoElement.hasAttribute("controls")) { + videoElement.removeAttribute("controls"); + } + if (!videoElement.hasAttribute("playsinline")) { + videoElement.setAttribute("playsinline", "true"); + } + if (videoElement.currentTime !== 0) { + // in case we previously played this video and are returning to this page... + videoElement.currentTime = 0; + } + }); } public HandlePageVisible(bloomPage: HTMLElement) { @@ -55,29 +54,13 @@ export class Video { this.currentVideoElement = undefined; return; } - this.currentVideoElement = this.getFirstVideo() as HTMLVideoElement; - this.videoEnded = false; - this.currentVideoElement.onended = (ev: Event) => { - this.videoEnded = true; - this.reportVideoPlayed( - (ev.target as HTMLVideoElement).currentTime - - this.videoStartTime - ); - if (this.PageVideoComplete) { - this.PageVideoComplete.raise({ - page: bloomPage, - video: this.currentVideoElement! - }); - } - }; if (currentPlaybackMode === PlaybackMode.VideoPaused) { - this.currentVideoElement.pause(); + this.currentVideoElement?.pause(); } else { - const videoElement = this.currentVideoElement; if (!isMacOrIOS()) { // Delay the start of the video by a little bit so the user can get oriented (BL-6985) window.setTimeout(() => { - this.playVideoCallback(videoElement, bloomPage); + this.playAllVideo(this.getVideoElements()); }, 1000); } else { // To auto-play a video w/sound on Apple Webkit browsers, the JS that invokes video.play() @@ -91,66 +74,41 @@ export class Video { // However, when I tried to detect audio tracks, it didn't detect the correct result // the first time a video with audio was loaded, even when I tried checking the readyState. // So, for now we'll just do this for all videos on Mac or iOS, even if they don't have audio tracks. - this.playVideoCallback(videoElement, bloomPage); + this.playAllVideo(this.getVideoElements()); } } } - private playVideoCallback( - videoElement: HTMLVideoElement, - bloomPage: HTMLElement - ) { - // When we go to a new page with a video on it, we delay 1 second to allow the user to get - // oriented to the new page. During that second, it's just possible that the user went on to - // another page. Better check and not play the video, if that's the case. - // Storybook "General video with audio" can be used to test this. - if (this.currentPage !== bloomPage) { - return; - } - this.videoStartTime = videoElement.currentTime; - const promise = videoElement.play(); - if (promise) { - promise.catch(reason => { - console.log(reason); - if (this.PageVideoComplete) { - this.PageVideoComplete.raise({ - page: bloomPage, - video: videoElement - }); - } - }); - } + private static getVideoElements(page: HTMLElement): HTMLVideoElement[] { + return Array.from(page.getElementsByClassName("bloom-videoContainer")) + .map(container => container.getElementsByTagName("video")[0]) + .filter(video => video !== undefined); } - - // At this point anyway, we just play the first video on the page. I think we'll have other UI problems - // to address before we get around to dealing with multiple videos on a page, so we'll just assume we play - // the first one for now. - private getFirstVideo(): HTMLVideoElement | undefined { - const videoContainers = this.currentPage.getElementsByClassName( - "bloom-videoContainer" - ); - if (videoContainers.length === 0) { - return undefined; - } - // There should only be one... but in any case we'll just play the first. - const container = videoContainers[0]; - const videoElements = container.getElementsByTagName("video"); - return videoElements.length === 0 ? undefined : videoElements[0]; + private getVideoElements(): HTMLVideoElement[] { + return Video.getVideoElements(this.currentPage); } public play() { if (currentPlaybackMode === PlaybackMode.VideoPlaying) { return; // no change. } - const videoElement = this.currentVideoElement; - if (!videoElement) { - return; // no change - } setCurrentPlaybackMode(PlaybackMode.VideoPlaying); - // If it has ended, it's going to replay from the beginning, even though - // (to prevent an abrupt visual effect) we didn't reset currentTime when it ended. - this.videoStartTime = this.videoEnded ? 0 : videoElement.currentTime; - videoElement.play(); + if (this.isPlayingSingleVideo) + this.playAllVideo([this.currentVideoElement!]); + else this.resumePlayAllVideo(); + } + + private resumePlayAllVideo() { + const allVideoElements = this.getVideoElements(); + let videoElements = allVideoElements; + if (this.currentVideoElement) { + // get subset of allVideoElements starting with currentVideoElement + const startIndex = allVideoElements.indexOf( + this.currentVideoElement + ); + videoElements = allVideoElements.slice(startIndex); + } + this.playAllVideo(videoElements); } public pause() { @@ -173,7 +131,7 @@ export class Video { ) { // It's playing, and we're about to stop it...report how long it's been going. this.reportVideoPlayed( - videoElement.currentTime - this.videoStartTime + videoElement.currentTime - this.currentVideoStartTime ); } videoElement.pause(); @@ -186,4 +144,57 @@ export class Video { public hidingPage() { this.pauseCurrentVideo(); // but don't set paused state. } + + public replaySingleVideo(video: HTMLVideoElement) { + this.isPlayingSingleVideo = true; + this.playAllVideo([video]); + } + + // Play the specified elements, one after the other. When the last completes, raise the PageVideoComplete event. + // + // Note, there is a very similar function in narration.ts. It would be nice to combine them, but + // this one must be here and must be part of the Video class so it can handle play/pause, analytics, etc. + public playAllVideo(elements: HTMLVideoElement[]) { + if (elements.length === 0) { + this.currentVideoElement = undefined; + this.isPlayingSingleVideo = false; + if (this.PageVideoComplete) { + this.PageVideoComplete.raise({ + page: this.currentPage, + videos: this.getVideoElements() + }); + } + return; + } + + const video = elements[0]; + + // If we somehow get into a state where the video is not on the current page, don't continue. + const pageForVideo = video.closest(".bloom-page") as HTMLDivElement; + if (this.currentPage !== pageForVideo) { + this.currentVideoElement = undefined; + return; + } + + this.currentVideoElement = video; + video.addEventListener( + "ended", + () => { + this.reportVideoPlayed( + video.currentTime - this.currentVideoStartTime + ); + this.playAllVideo(elements.slice(1)); + }, + { once: true } + ); + setCurrentPlaybackMode(PlaybackMode.VideoPlaying); + this.currentVideoStartTime = video.currentTime || 0; + const promise = video.play(); + + // If there is an error, try to continue with the next video. + promise?.catch(reason => { + console.error("Video play failed", reason); + this.playAllVideo(elements.slice(1)); + }); + } }