From a4dd205976f345e9082ce917ba4600237ca0018c Mon Sep 17 00:00:00 2001 From: John Thomson Date: Fri, 12 Jul 2024 15:25:23 -0500 Subject: [PATCH 1/2] Implement drag activities Note that this includes adding a story for an example book, but not its actual content. That's a lot of files, many of them binary. Cleaning up audio Getting rid of narration.ts Reorder narration.ts to original. Merge with narrationUtils Move dragActivityRuntime.ts (without changing it, to maximize the chances of history recognizing it as a move) WIP switch out dragActivityNarration.ts WIP on Bloom games 11/30/23 Appears I was trying to get narration sorted out. Not sure how far I got. All the stuff that I think needs to change to replace dragActivityNarration Replace dragActivityNarration with modified narration.ts Also bring dragActivityRuntime up to date with Bloom desktop Post-rebase debugging --- package.json | 1 + src/activities/ActivityContext.ts | 12 +- src/activities/activityManager.ts | 22 + .../dragActivities/DragToDestination.ts | 82 + src/bloom-player-core.tsx | 191 +- src/dragActivityRuntime.ts | 1108 +++++++++ src/event.ts | 8 +- src/music.ts | 5 +- src/narration.test.ts | 2 +- src/narration.ts | 2017 +++++++++-------- src/narrationUtils.ts | 1134 +++++++++ src/stories/index.tsx | 4 + src/video.ts | 17 +- 13 files changed, 3553 insertions(+), 1050 deletions(-) create mode 100644 src/activities/dragActivities/DragToDestination.ts create mode 100644 src/dragActivityRuntime.ts diff --git a/package.json b/package.json index bc3d48ae..2b21fa9c 100644 --- a/package.json +++ b/package.json @@ -110,6 +110,7 @@ "dist/*.css", "dist/*.tsv" ], + "packageManager": "yarn@1.22.19", "volta": { "node": "16.14.0", "yarn": "1.22.19" diff --git a/src/activities/ActivityContext.ts b/src/activities/ActivityContext.ts index ee276544..740887b6 100644 --- a/src/activities/ActivityContext.ts +++ b/src/activities/ActivityContext.ts @@ -98,7 +98,7 @@ export class ActivityContext { public playSound(url) { const player = this.getPagePlayer(); - player.setAttribute("src", url); + player.setAttribute("src", url.default); player.play(); } @@ -138,6 +138,16 @@ export class ActivityContext { target.addEventListener(name, listener, options); } + public removeEventListener( + name: string, + target: Element, + listener: EventListener, + options?: AddEventListenerOptions | undefined + ) { + // we could try to remove it from this.listeners, but it's harmless to remove it again + target.removeEventListener(name, listener, options); + } + // this is called by the activity manager after it stops the activity. public stop() { // detach all the listeners diff --git a/src/activities/activityManager.ts b/src/activities/activityManager.ts index 979abb05..4d464e3a 100644 --- a/src/activities/activityManager.ts +++ b/src/activities/activityManager.ts @@ -3,6 +3,7 @@ import { ActivityContext } from "./ActivityContext"; const iframeModule = require("./iframeActivity.ts"); const simpleDomChoiceActivityModule = require("./domActivities/SimpleDomChoice.ts"); const simpleCheckboxQuizModule = require("./domActivities/SimpleCheckboxQuiz.ts"); +const dragToDestinationModule = require("./dragActivities/DragToDestination.ts"); // This is the module that the activity has to implement (the file must export these functions) export interface IActivityModule { @@ -35,6 +36,7 @@ export interface IActivityRequirements { dragging?: boolean; clicking?: boolean; typing?: boolean; + soundManagement?: boolean; // suppress normal sound (and music, and animation) } // This is the object (implemented by us, not the activity) that represents our own @@ -60,6 +62,20 @@ export class ActivityManager { this.builtInActivities[ simpleCheckboxQuizModule.dataActivityID ] = simpleCheckboxQuizModule as IActivityModule; + this.builtInActivities[ + "drag-to-destination" + ] = dragToDestinationModule as IActivityModule; + // Review: currently these two use the same module. A lot of stuff is shared, all the way down to the + // prepareActivity() function in dragActivityRuntime. But a good many specialized TOP types are + // specific to one of the three and not needed for the others. It may be helpful to tease things + // apart more, for example, three separate implementations of IActivityModule and PrepareActivity + // which call common code for the setup tasks common to all three. + this.builtInActivities[ + "sort-sentence" + ] = dragToDestinationModule as IActivityModule; + this.builtInActivities[ + "word-chooser-slider" + ] = dragToDestinationModule as IActivityModule; } public getActivityAbsorbsDragging(): boolean { return ( @@ -78,6 +94,12 @@ export class ActivityManager { !!this.currentActivity && !!this.currentActivity.requirements.typing ); } + public getActivityManagesSound(): boolean { + return ( + !!this.currentActivity && + !!this.currentActivity.requirements.soundManagement + ); + } private currentActivity: IActivityInformation | undefined; private loadedActivityScripts: { [name: string]: IActivityInformation; diff --git a/src/activities/dragActivities/DragToDestination.ts b/src/activities/dragActivities/DragToDestination.ts new file mode 100644 index 00000000..fee893fa --- /dev/null +++ b/src/activities/dragActivities/DragToDestination.ts @@ -0,0 +1,82 @@ +import { ActivityContext } from "../ActivityContext"; +import { IActivityObject, IActivityRequirements } from "../activityManager"; +import { + prepareActivity, + undoPrepareActivity +} from "../../dragActivityRuntime"; +// tslint:disable-next-line: no-submodule-imports +/* Not using. See comment below: + const activityCss = require("!!raw-loader!./multipleChoiceDomActivity.css") + .default;*/ + +// This class is intentionally very generic. All it needs is that the html of the +// page it is given should have some objects (typically bloom-textOverPicture) that have +// data-correct-position. These objects are made draggable...[Todo: document more of this as we implement] + +// Note that you won't find any code using this directly. Instead, +// it gets used by the ActivityManager as the default export of this module. +export default class DragToDestinationActivity implements IActivityObject { + private activityContext: ActivityContext; + // When a page that has this activity becomes the selected one, the bloom-player calls this. + // We need to connect any listeners, start animation, etc. Here, + // we are using a javascript class to make sure that we get a fresh start, + // which is important because the user could be either + // coming back to this page, or going to another instance of this activity + // in a subsequent page. + // eslint-disable-next-line no-unused-vars + public constructor(public pageElement: HTMLElement) {} + + public showingPage(activityContext: ActivityContext) { + this.activityContext = activityContext; + this.prepareToDisplayActivityEachTime(activityContext); + } + + // Do just those things that we only want to do once per read of the book. + // In the current implementation of activityManager, this is operating on a copy of the page html, + // NOT the real DOM the user will eventually interact with. + public initializePageHtml(activityContext: ActivityContext) {} + + // The context removes event listeners each time the page is shown, so we have to put them back. + private prepareToDisplayActivityEachTime(activityContext: ActivityContext) { + this.activityContext = activityContext; + // These classes are added one layer outside the page body. This is an element that is a wrapper + // for our scoped styles...the furthest out element we can put classes on and have them work + // properly with scoped styles. It's a good place to put classes that affect the state of everything + // in the page. + activityContext.pageElement.parentElement?.classList.add( + "drag-activity-try-it", + "drag-activity-start" + ); + prepareActivity(activityContext.pageElement, next => { + // Move to the next or previous page. + if (next) { + activityContext.navigateToNextPage(); + } else { + activityContext.navigateToPreviousPage(); + } + }); + } + + // When our page is not the selected one, the bloom-player calls this. + // It will also tell our context to stop, which will disconnect the listeners we registered with it + public stop() { + if (this.activityContext) { + undoPrepareActivity(this.activityContext.pageElement); + this.activityContext.pageElement.parentElement?.classList.remove( + "drag-activity-try-it", + "drag-activity-start", + "drag-activity-correct", + "drag-activity-wrong" + ); + } + } +} + +export function activityRequirements(): IActivityRequirements { + return { + dragging: true, // this activity is all about dragging things around, we don't want dragging to change pages + clicking: true, // not sure we need this, but can we actually support dragging without supporting clicking? + typing: false, + soundManagement: true // many sounds played only after specific events. + }; +} diff --git a/src/bloom-player-core.tsx b/src/bloom-player-core.tsx index e0cd4d0f..e18f2d16 100644 --- a/src/bloom-player-core.tsx +++ b/src/bloom-player-core.tsx @@ -13,7 +13,6 @@ import "swiper/dist/css/swiper.min.css"; import "./bloom-player-ui.less"; import "./bloom-player-content.less"; import "./bloom-player-pre-appearance-system-book.less"; -import Narration from "./narration"; import LiteEvent from "./event"; import { Animation } from "./animation"; import { IPageVideoComplete, Video } from "./video"; @@ -52,17 +51,29 @@ import { kLocalStorageBookUrlKey } from "./bloomPlayerAnalytics"; import { autoPlayType } from "./bloom-player-controls"; - -export enum PlaybackMode { - NewPage, // starting a new page ready to play - NewPageMediaPaused, // starting a new page in the "paused" state - VideoPlaying, // video is playing - VideoPaused, // video is paused - AudioPlaying, // narration and/or animation are playing (or possibly finished) - AudioPaused, // narration and/or animation are paused - MediaFinished // video, narration, and/or animation has played (possibly no media to play) - // Note that music can be playing when the state is either AudioPlaying or MediaFinished. -} +import { setCurrentPage } from "./narration"; +import { + currentPlaybackMode, + setCurrentPlaybackMode, + PlaybackMode, + listenForPlayDuration, + setTestIsSwipeInProgress, + setLogNarration, + setPlayerUrlPrefix, + computeDuration, + PageNarrationComplete, + PlayFailed, + PlayCompleted, + ToggleImageDescription, + pause, + getCurrentPage, + play, + hidingPage, + pageHasAudio, + setIncludeImageDescriptions, + playAllSentences +} from "./narration"; +import { logSound } from "./videoRecordingSupport"; // BloomPlayer takes a URL param that directs it to Bloom book. // (See comment on sourceUrl for exactly how.) // It displays pages from the book and allows them to be turned by dragging. @@ -276,7 +287,6 @@ export class BloomPlayerCore extends React.Component { private metaDataObject: any | undefined; private htmlElement: HTMLHtmlElement | undefined; - private narration: Narration; private animation: Animation; private music: Music; private video: Video; @@ -289,8 +299,6 @@ export class BloomPlayerCore extends React.Component { private static currentPageHasVideo: boolean; private currentPageHidesNavigationButtons: boolean = false; - public static currentPlaybackMode: PlaybackMode; - private indexOflastNumberedPage: number; public componentDidMount() { @@ -465,12 +473,18 @@ export class BloomPlayerCore extends React.Component { ? this.sourceUrl : this.sourceUrl + "/" + filename + ".htm"; - this.music.urlPrefix = this.narration.urlPrefix = this.urlPrefix = haveFullPath + let urlPrefixT = haveFullPath ? this.sourceUrl.substring( 0, Math.max(slashIndex, encodedSlashIndex) ) : this.sourceUrl; + if (!urlPrefixT.startsWith("http")) { + // Only in storybook with local books? + urlPrefixT = window.location.origin + "/" + urlPrefixT; + } + this.music.urlPrefix = this.urlPrefix = urlPrefixT; + setPlayerUrlPrefix(this.music.urlPrefix); // Note: this does not currently seem to work when using the storybook fileserver. // I hypothesize that it automatically filters files starting with a period, // so asking for .distribution fails even if the local book folder (e.g., Testing @@ -530,6 +544,10 @@ export class BloomPlayerCore extends React.Component { } this.animation.PlayAnimations = this.bookInfo.playAnimations; + console.log( + "animation.PlayAnimations", + this.animation.PlayAnimations + ); this.collectBodyAttributes(body); this.makeNonEditable(body); @@ -1088,15 +1106,6 @@ export class BloomPlayerCore extends React.Component { } }; - private handlePageDurationAvailable = ( - pageElement: HTMLElement | undefined - ) => { - this.animation.HandlePageDurationAvailable( - pageElement!, - this.narration.PageDuration - ); - }; - private handlePlayFailed = () => { this.setState({ inPauseForced: true }); if (this.props.setForcedPausedCallback) { @@ -1105,8 +1114,8 @@ export class BloomPlayerCore extends React.Component { }; private handlePlayCompleted = () => { - BloomPlayerCore.currentPlaybackMode = PlaybackMode.MediaFinished; - this.props.imageDescriptionCallback(false); + setCurrentPlaybackMode(PlaybackMode.MediaFinished); + this.props.imageDescriptionCallback(false); // whether or not we were before, now we're certainly not playing one. }; private handleToggleImageDescription = (inImageDescription: boolean) => { @@ -1124,26 +1133,29 @@ export class BloomPlayerCore extends React.Component { this.handlePageVideoComplete ); } - if (!this.narration) { - this.narration = new Narration(); - this.narration.PageDurationAvailable = new LiteEvent(); - this.narration.PageNarrationComplete = new LiteEvent(); - this.narration.PlayFailed = new LiteEvent(); - this.narration.PlayCompleted = new LiteEvent(); - this.narration.ToggleImageDescription = new LiteEvent(); + if (!this.animation) { this.animation = new Animation(); - this.narration.PageDurationAvailable.subscribe( - this.handlePageDurationAvailable - ); - this.narration.PageNarrationComplete.subscribe( - this.handlePageNarrationComplete - ); - this.narration.PlayFailed.subscribe(this.handlePlayFailed); - this.narration.PlayCompleted.subscribe(this.handlePlayCompleted); - this.narration.ToggleImageDescription.subscribe( - this.handleToggleImageDescription - ); } + + PageNarrationComplete.subscribe(this.handlePageNarrationComplete); + PlayFailed.subscribe(this.handlePlayFailed); + PlayCompleted.subscribe(this.handlePlayCompleted); + listenForPlayDuration(this.storeAudioAnalytics.bind(this)); + ToggleImageDescription.subscribe( + this.handleToggleImageDescription.bind(this) + ); + // allows narration to ask whether swiping to this page is still in progress. + setTestIsSwipeInProgress(() => { + console.log( + "animating: " + + this.swiperInstance + + " " + + this.swiperInstance?.animating + ); + return this.swiperInstance?.animating; + }); + setLogNarration(url => logSound(url, 1)); + if (!this.music) { this.music = new Music(); this.music.PlayFailed = new LiteEvent(); @@ -1182,25 +1194,23 @@ export class BloomPlayerCore extends React.Component { } private handlePausePlay() { + // props indicates the state we want to be in, typically from the BloomPlayerControls state. + // Calling this method indicates we are not in that state, so we need to change it to match. if (this.props.paused) { this.pauseAllMultimedia(); } else { // This test determines if we changed pages while paused, // since the narration object won't yet be updated. if ( - BloomPlayerCore.currentPage !== this.narration.playerPage || - BloomPlayerCore.currentPlaybackMode === - PlaybackMode.MediaFinished + BloomPlayerCore.currentPage !== getCurrentPage() || + currentPlaybackMode === PlaybackMode.MediaFinished ) { this.resetForNewPageAndPlay(BloomPlayerCore.currentPage!); } else { - if ( - BloomPlayerCore.currentPlaybackMode === - PlaybackMode.VideoPaused - ) { + if (currentPlaybackMode === PlaybackMode.VideoPaused) { this.video.play(); // sets currentPlaybackMode = VideoPlaying } else { - this.narration.play(); // sets currentPlaybackMode = AudioPlaying + play(); // sets currentPlaybackMode = AudioPlaying this.animation.PlayAnimation(); this.music.play(); } @@ -1420,26 +1430,18 @@ export class BloomPlayerCore extends React.Component { private unsubscribeAllEvents() { this.video.PageVideoComplete.unsubscribe(this.handlePageVideoComplete); - this.narration.PageDurationAvailable.unsubscribe( - this.handlePageDurationAvailable - ); - this.narration.PageNarrationComplete.unsubscribe( - this.handlePageNarrationComplete - ); - this.narration.PlayFailed.unsubscribe(this.handlePlayFailed); - this.narration.PlayCompleted.unsubscribe(this.handlePlayCompleted); - this.narration.ToggleImageDescription.unsubscribe( - this.handleToggleImageDescription - ); + PageNarrationComplete.unsubscribe(this.handlePageNarrationComplete); + PlayFailed.unsubscribe(this.handlePlayFailed); + PlayCompleted.unsubscribe(this.handlePlayCompleted); + ToggleImageDescription.unsubscribe(this.handleToggleImageDescription); } private pauseAllMultimedia() { - if (BloomPlayerCore.currentPlaybackMode === PlaybackMode.VideoPlaying) { + const temp = currentPlaybackMode; + if (currentPlaybackMode === PlaybackMode.VideoPlaying) { this.video.pause(); // sets currentPlaybackMode = VideoPaused - } else if ( - BloomPlayerCore.currentPlaybackMode === PlaybackMode.AudioPlaying - ) { - this.narration.pause(); // sets currentPlaybackMode = AudioPaused + } else if (currentPlaybackMode === PlaybackMode.AudioPlaying) { + pause(); // sets currentPlaybackMode = AudioPaused this.animation.PauseAnimation(); } // Music keeps playing after all video, narration, and animation have finished. @@ -1959,7 +1961,7 @@ export class BloomPlayerCore extends React.Component { ); } - this.narration.setIncludeImageDescriptions( + setIncludeImageDescriptions( this.props.shouldReadImageDescriptions && this.hasImageDescriptions ); const swiperParams: any = { @@ -2339,7 +2341,7 @@ export class BloomPlayerCore extends React.Component { if (!page) { return; } - BloomPlayerCore.currentPlaybackMode = PlaybackMode.MediaFinished; + setCurrentPlaybackMode(PlaybackMode.MediaFinished); // TODO: at this point, signal BloomPlayerControls to switch the pause button to show play. if (this.shouldAutoPlay()) { @@ -2386,7 +2388,6 @@ export class BloomPlayerCore extends React.Component { //console.log("aborting setIndex because still starting up"); return; } - clearTimeout(this.narration.pageNarrationCompleteTimer); this.setState({ currentSwiperIndex: index }); const bloomPage = this.getPageAtSwiperIndex(index); if (bloomPage) { @@ -2399,17 +2400,15 @@ export class BloomPlayerCore extends React.Component { // its continued playing. this.video.hidingPage(); this.video.HandlePageBeforeVisible(bloomPage); - this.narration.hidingPage(); + hidingPage(); this.music.hidingPage(); if ( - BloomPlayerCore.currentPlaybackMode === - PlaybackMode.AudioPaused || - BloomPlayerCore.currentPlaybackMode === PlaybackMode.VideoPaused + currentPlaybackMode === PlaybackMode.AudioPaused || + currentPlaybackMode === PlaybackMode.VideoPaused ) { - BloomPlayerCore.currentPlaybackMode = - PlaybackMode.NewPageMediaPaused; + setCurrentPlaybackMode(PlaybackMode.NewPageMediaPaused); } else { - BloomPlayerCore.currentPlaybackMode = PlaybackMode.NewPage; + setCurrentPlaybackMode(PlaybackMode.NewPage); } } } @@ -2515,7 +2514,7 @@ export class BloomPlayerCore extends React.Component { if (this.props.reportPageProperties) { // Informs containing react controls (in the same frame) this.props.reportPageProperties({ - hasAudio: this.narration.pageHasAudio(bloomPage), + hasAudio: pageHasAudio(bloomPage), hasMusic: this.music.pageHasMusic(bloomPage), hasVideo: BloomPlayerCore.currentPageHasVideo }); @@ -2742,26 +2741,25 @@ export class BloomPlayerCore extends React.Component { } // called by narration.ts - public static storeAudioAnalytics(duration: number): void { + public storeAudioAnalytics(duration: number): void { if (duration < 0.001 || Number.isNaN(duration)) { return; } - const player = BloomPlayerCore.currentPagePlayer; - player.bookInteraction.totalAudioDuration += duration; + this.bookInteraction.totalAudioDuration += duration; - if (player.isXmatterPage()) { + if (this.isXmatterPage()) { // Our policy is only to count non-xmatter audio pages. BL-7334. return; } - if (!player.bookInteraction.reportedAudioOnCurrentPage) { - player.bookInteraction.reportedAudioOnCurrentPage = true; - player.bookInteraction.audioPageShown( + if (!this.bookInteraction.reportedAudioOnCurrentPage) { + this.bookInteraction.reportedAudioOnCurrentPage = true; + this.bookInteraction.audioPageShown( BloomPlayerCore.currentPageIndex ); } - player.sendUpdateOfBookProgressReportToExternalContext(); + this.sendUpdateOfBookProgressReportToExternalContext(); } public static storeVideoAnalytics(duration: number) { @@ -2800,9 +2798,10 @@ export class BloomPlayerCore extends React.Component { } } this.animation.PlayAnimation(); // get rid of classes that made it pause + setCurrentPage(bloomPage); // State must be set before calling HandlePageVisible() and related methods. if (BloomPlayerCore.currentPageHasVideo) { - BloomPlayerCore.currentPlaybackMode = PlaybackMode.VideoPlaying; + setCurrentPlaybackMode(PlaybackMode.VideoPlaying); this.video.HandlePageVisible(bloomPage); this.music.pause(); // in case we have audio from previous page } else { @@ -2813,17 +2812,19 @@ export class BloomPlayerCore extends React.Component { sentBloomNotification: boolean = false; public playAudioAndAnimation(bloomPage: HTMLElement | undefined) { - BloomPlayerCore.currentPlaybackMode = PlaybackMode.AudioPlaying; + if (this.activityManager.getActivityManagesSound()) { + return; // we don't just want to play them all, the activity code will do it selectively. + } + setCurrentPlaybackMode(PlaybackMode.AudioPlaying); if (!bloomPage) return; - this.narration.setSwiper(this.swiperInstance); - // When we have computed it, this will raise PageDurationComplete, - // which calls an animation method to start the image animation. - this.narration.computeDuration(bloomPage); + + const duration = computeDuration(bloomPage); + this.animation.HandlePageDurationAvailable(bloomPage!, duration); // Tail end of the method, happens at once if we're not posting, only after // the post completes if we are. const finishUp = () => { - this.narration.playAllSentences(bloomPage); + playAllSentences(bloomPage); if (Animation.pageHasAnimation(bloomPage as HTMLDivElement)) { this.animation.HandlePageBeforeVisible(bloomPage); } diff --git a/src/dragActivityRuntime.ts b/src/dragActivityRuntime.ts new file mode 100644 index 00000000..d02fd5a1 --- /dev/null +++ b/src/dragActivityRuntime.ts @@ -0,0 +1,1108 @@ +// This is the code that is shared between the Play tab of the bloom games +// (also known as drag activities) and bloom-player. +// It wants to live in the dragActivity folder because it is specific to drag activities. +// However, it also wants to live in the same place relative to narration.ts both in +// bloom player and bloom desktop. For now that's a stronger requirement. +// For now, in Bloom desktop, this file is only used in the Play tab of drag activities, +// so both live there. In Bloom player, both live in the root src directory. +// In the long run, the answer is probably a folder, or even an npm package, for all +// the stuff that the two progams share...or maybe we can make bloom player publish +// these files along with the output bundle and have bloom desktop use them from there. +// For now, though, it's much easier to just edit them and have them built automatically +// than to have this code in another repo. + +import { get } from "jquery"; +import { + kAudioSentence, + playAllAudio, + playAllVideo, + urlPrefix +} from "./narration"; + +let targetPositions: { x: number; y: number }[] = []; +let originalPositions = new Map(); +let currentPage: HTMLElement | undefined; +// Action to invoke if the user clicks a change page button. +// Our latest templates don't have their own change page buttons, just encourage the user +// to leave room for the player to add them. +let currentChangePageAction: (next: boolean) => void | undefined; +let positionsToRestore: { x: string; y: string; elt: HTMLElement }[] = []; + +// Save the current positions of all draggables (when entering Play tab, so we can restore them when leaving). +const savePositions = (page: HTMLElement) => { + positionsToRestore = []; + page.querySelectorAll("[data-bubble-id]").forEach((elt: HTMLElement) => { + positionsToRestore.push({ + x: elt.style.left, + y: elt.style.top, + elt + }); + }); +}; +// Restore the positions saved by savePositions (when leaving the Play tab, or leaving this page altogether +// after being in that tab). +const restorePositions = () => { + positionsToRestore.forEach(p => { + p.elt.style.left = p.x; + p.elt.style.top = p.y; + }); + // In case we do more editing after leaving the Play tab, we don't want to restore the same positions again + // if we leave the page completely. + positionsToRestore = []; +}; + +// Function to call to get everything ready for playing the game. +// Things that get done here should usually be undone in undoPrepareActivity. +export function prepareActivity( + page: HTMLElement, + // Possibly obsolete: an action to take when the user clicks a change page button. + // Current plan is to just let BP add its own change page buttons. + changePageAction: (next: boolean) => void +) { + currentPage = page; + currentChangePageAction = changePageAction; + // not sure we need this in BP, but definitely for when Bloom desktop goes to another tab. + savePositions(page); + + // Set up event listeners for any change page buttons. + const changePageButtons = Array.from( + page.getElementsByClassName("bloom-change-page-button") + ); + changePageButtons.forEach(b => + b.addEventListener("click", changePageButtonClicked) + ); + + // Hide image titles, which might give too much away, or distract. + Array.from(document.getElementsByClassName("bloom-imageContainer")).forEach( + container => { + (container as HTMLElement).title = ""; + } + ); + + // Record the positions of targets as snap locations and the original positions of draggables. + // Add event listeners to draggables to start dragging. + targetPositions = []; + originalPositions = new Map(); + const draggables = Array.from(page.querySelectorAll("[data-bubble-id]")); + const targets: HTMLElement[] = []; + draggables.forEach((elt: HTMLElement) => { + const targetId = elt.getAttribute("data-bubble-id"); + const target = page.querySelector( + `[data-target-of="${targetId}"]` + ) as HTMLElement; + if (target) { + const x = target.offsetLeft; + const y = target.offsetTop; + targetPositions.push({ x, y }); + targets.push(target); + } + // if it has data-bubble-id, it should be draggable, just not needed + // for the right answer. + originalPositions.set(elt, { x: elt.offsetLeft, y: elt.offsetTop }); + elt.addEventListener("pointerdown", startDrag, { capture: true }); + }); + + // Add event listeners to (other) text items that should play audio when clicked. + const dontPlayWhenClicked = draggables.concat(targets); + const otherTextItems = Array.from( + page.getElementsByClassName("bloom-visibility-code-on") + ).filter(e => { + var top = e.closest(".bloom-textOverPicture") as HTMLElement; + if (!top) { + // don't think this can happen with current game templates, + // but if there's some other text on the page, may as well play when clicked + // if it can. + return true; + } + // draggables play as well as doing more complex things when clicked. + // targets don't need to play. + return dontPlayWhenClicked.indexOf(top) < 0; + }); + otherTextItems.forEach(e => { + e.addEventListener("pointerdown", playAudioOfTarget); + }); + + // Add event listeners to check, try again, and show correct buttons. + const checkButtons = Array.from( + page.getElementsByClassName("check-button") + ); + const tryAgainButtons = Array.from( + page.getElementsByClassName("try-again-button") + ); + const showCorrectButtons = Array.from( + page.getElementsByClassName("show-correct-button") + ); + + checkButtons.forEach((elt: HTMLElement) => { + elt.addEventListener("click", performCheck); + }); + tryAgainButtons.forEach((elt: HTMLElement) => { + elt.addEventListener("click", performTryAgain); + }); + showCorrectButtons.forEach((elt: HTMLElement) => { + elt.addEventListener("click", showCorrect); + }); + + prepareOrderSentenceActivity(page); + + // Slider: // for drag-word-chooser-slider + // setupWordChooserSlider(page); + // setSlideablesVisibility(page, false); + // // We may not want to immediately play the first word, because we may want to play some + // // other stuff first. So we call this (before playInitialElements, so the word is given + // // the bloom-activeTextBox class) but tell it not to play. + // // The element with bloom-activeTextBox is put into the list of things that playInitialElements + // // will play, after everything else that should happen initially. + + // showARandomWord(page, false); + // setupSliderImageEvents(page); + + playInitialElements(page); +} + +// Break any order-sentence element into words and +// randomize word order in sentence for reader to sort +const prepareOrderSentenceActivity = (page: HTMLElement) => { + Array.from(page.getElementsByClassName("drag-item-order-sentence")).forEach( + (elt: HTMLElement) => { + const contentElt = elt.getElementsByClassName( + "bloom-content1" + )[0] as HTMLElement; + const content = contentElt?.textContent?.trim(); + if (!content) return; + const words = content.split(" "); + const shuffledWords = shuffle(words); + const container = page.ownerDocument.createElement("div"); + container.classList.add("drag-item-random-sentence"); + container.setAttribute("data-answer", content); + makeWordItems(page, shuffledWords, container, contentElt, true); + container.style.left = elt.style.left; + container.style.top = elt.style.top; + container.style.width = + elt.parentElement!.offsetWidth - elt.offsetLeft - 10 + "px"; + // Enhance: limit width somehow so it does not collide with other elements? + // Maybe now we tweaked word padding to make the original sentence take up more + // space, we could use its own width? + elt.parentElement?.insertBefore(container, elt); + } + ); +}; + +// Cleans up whatever prepareACtivity() did, especially when switching to another tab. +// May also be useful to do when switching pages in player. If not, we may want to move +// this out of this runtime file; but it's nice to keep it with prepareActivity. +export function undoPrepareActivity(page: HTMLElement) { + restorePositions(); + const changePageButtons = Array.from( + page.getElementsByClassName("bloom-change-page-button") + ); + changePageButtons.forEach(b => + b.removeEventListener("click", changePageButtonClicked) + ); + + Array.from(page.getElementsByClassName("bloom-visibility-code-on")).forEach( + e => { + e.removeEventListener("pointerdown", playAudioOfTarget); + } + ); + + page.querySelectorAll("[data-bubble-id]").forEach((elt: HTMLElement) => { + elt.removeEventListener("pointerdown", startDrag, { capture: true }); + }); + const checkButtons = Array.from( + page.getElementsByClassName("check-button") + ); + const tryAgainButtons = Array.from( + page.getElementsByClassName("try-again-button") + ); + const showCorrectButtons = Array.from( + page.getElementsByClassName("show-correct-button") + ); + + checkButtons.forEach((elt: HTMLElement) => { + elt.removeEventListener("click", performCheck); + }); + showCorrectButtons.forEach((elt: HTMLElement) => { + elt.removeEventListener("click", showCorrect); + }); + tryAgainButtons.forEach((elt: HTMLElement) => { + elt.removeEventListener("click", performTryAgain); + }); + + Array.from( + page.getElementsByClassName("drag-item-random-sentence") + ).forEach((elt: HTMLElement) => { + elt.parentElement?.removeChild(elt); + }); + //Slider: setSlideablesVisibility(page, true); + // Array.from(page.getElementsByTagName("img")).forEach((img: HTMLElement) => { + // img.removeEventListener("click", clickSliderImage); + // }); +} + +const playAudioOfTarget = (e: PointerEvent) => { + const target = e.currentTarget as HTMLElement; + playAudioOf(target); +}; + +const playAudioOf = (element: HTMLElement) => { + const possibleElements = getVisibleEditables(element); + const playables = getAudioSentences(possibleElements); + playAllAudio(playables, getPage(element)); +}; + +const getPage = (element: HTMLElement): HTMLElement => { + return element.closest(".bloom-page") as HTMLElement; +}; + +function makeWordItems( + page: HTMLElement, + words: string[], + container: HTMLElement, + // Something that has the right user-defined style class to apply to the words. + // May be a bloom-content1 child of the original sentence, or a word item + // previously created by this function. + contentElt: HTMLElement, + // Should the reader be able to drag the words? Not when we're using this + // to show the correct answer. + makeDraggable: boolean +) { + const userStyle = + Array.from(contentElt?.classList)?.find(c => c.endsWith("-style")) ?? + "Normal-style"; + words.forEach(word => { + const wordItem = page.ownerDocument.createElement("div"); + wordItem.classList.add("drag-item-order-word"); + wordItem.textContent = word; + container.appendChild(wordItem); + wordItem.classList.add(userStyle); + if (makeDraggable) { + wordItem.addEventListener("pointerdown", startDragWordInSentence); + } + }); +} + +function changePageButtonClicked(e: MouseEvent) { + const next = (e.currentTarget as HTMLElement).classList.contains( + "bloom-next-page" + ); + currentChangePageAction?.(next); +} + +function playInitialElements(page: HTMLElement) { + const initialFilter = e => { + const top = e.closest(".bloom-textOverPicture") as HTMLElement; + if (!top) { + // not an overlay at all. (Note that all overlays have this class, including + // video and image overlays.) Maybe not possible in a drag-activity, but just in case + return false; + } + if (top.classList.contains("draggable-text")) { + return false; // draggable items are played only when clicked + } + if (top.classList.contains("drag-item-order-sentence")) { + return false; // This would give away the answer + } + if (top.classList.contains("bloom-wordChoice")) { + return false; // Only one of these should be played, after any instructions + } + // This might be redundant since they are not visible, but just in case + if ( + top.classList.contains("drag-item-correct") || + top.classList.contains("drag-item-wrong") + ) { + return false; // These are only played after they become visible + } + return true; + }; + const videoElements = Array.from(page.getElementsByTagName("video")).filter( + initialFilter + ); + const audioElements = getVisibleEditables(page).filter(initialFilter); + + //Slider: // This is used in drag-word-chooser-slider to mark the text item the user is currently + // // finding a matching image for. In that activity, it should be played last (after + // // the instructions.) + // const activeTextBox = page.getElementsByClassName( + // "bloom-activeTextBox" + // )[0] as HTMLElement; + // if (activeTextBox) { + // audioElements.push(activeTextBox); + // } + const playables = getAudioSentences(audioElements); + playAllVideo(videoElements, () => playAllAudio(playables, page)); +} + +function getAudioSentences(editables: HTMLElement[]) { + // Could be done more cleanly with flatMap or flat() but not ready to switch to es2019 yet. + const result: HTMLElement[] = []; + editables.forEach(e => { + if (e.classList.contains(kAudioSentence)) { + result.push(e); + } + result.push( + ...(Array.from( + e.getElementsByClassName(kAudioSentence) + ) as HTMLElement[]) + ); + }); + return result; +} + +function getVisibleEditables(container: HTMLElement) { + // We want to play any audio we have from divs the user can see. + // This is a crude test, but currently we always use display:none to hide unwanted languages. + const result = Array.from( + container.getElementsByClassName("bloom-editable") + ).filter( + e => window.getComputedStyle(e).display !== "none" + ) as HTMLElement[]; + if ( + container.classList.contains("bloom-editable") && + window.getComputedStyle(container).display !== "none" + ) { + result.push(container); + } + return result; +} + +function shuffle(array: T[]): T[] { + // review: something Copliot came up with. Is it guaranteed to be sufficiently different + // from the correct answer? + let currentIndex = array.length, + randomIndex; + while (0 !== currentIndex) { + randomIndex = Math.floor(Math.random() * currentIndex); + currentIndex--; + [array[currentIndex], array[randomIndex]] = [ + array[randomIndex], + array[currentIndex] + ]; + } + return array; +} + +// Put the page into the mode that shows the correct answers. +const showCorrect = (e: MouseEvent) => { + if (!currentPage) { + return; // huh?? but makes TS happy + } + currentPage + .querySelectorAll("[data-bubble-id]") + .forEach((elt: HTMLElement) => { + const targetId = elt.getAttribute("data-bubble-id"); + const target = currentPage?.querySelector( + `[data-target-of="${targetId}"]` + ) as HTMLElement; + if (!target) { + return; // this one is not required to be in a right place + } + const x = target.offsetLeft; + const y = target.offsetTop; + elt.style.left = x + "px"; + elt.style.top = y + "px"; + }); + Array.from( + currentPage.getElementsByClassName("drag-item-random-sentence") + ).forEach((container: HTMLElement) => { + const correctAnswer = + container.getAttribute("data-answer")?.split(" ") ?? []; + const userStyleSource = container.children[0] as HTMLElement; // before we wipe them! + container.innerHTML = ""; + makeWordItems( + currentPage!, + correctAnswer, + container, + userStyleSource, + false + ); + }); + classSetter(currentPage!, "drag-activity-wrong", false); + classSetter(currentPage!, "drag-activity-solution", true); +}; + +// where the mouse started the drag, relative to the top left of dragTarget +let dragStartX = 0; +let dragStartY = 0; +let dragTarget: HTMLElement; +let snapped = false; + +// Bloom desktop has a function getScale, but we do NOT want to use that here +// because it is not available in Bloom Reader and we don't want to add a dependency. +// So we define our own. +const getScale = (page: HTMLElement) => + page.getBoundingClientRect().width / page.offsetWidth; + +const startDrag = (e: PointerEvent) => { + if (e.button !== 0) return; // only left button + if (e.ctrlKey) return; // ignore ctrl+click + e.preventDefault(); // e.g., don't do default drag of child image + const target = e.currentTarget as HTMLElement; + dragTarget = target; + const page = target.closest(".bloom-page") as HTMLElement; + const scale = getScale(page); + // get the mouse cursor position at startup relative to the top left. + dragStartX = e.clientX / scale - target.offsetLeft; + dragStartY = e.clientY / scale - target.offsetTop; + target.setPointerCapture(e.pointerId); + target.addEventListener("pointerup", stopDrag); + target.addEventListener("pointermove", elementDrag); + playAudioOf(target); +}; + +const elementDrag = (e: PointerEvent) => { + const page = dragTarget.closest(".bloom-page") as HTMLElement; + const scale = getScale(page); + e.preventDefault(); + let x = e.clientX / scale - dragStartX; + let y = e.clientY / scale - dragStartY; + let deltaMin = Number.MAX_VALUE; + snapped = false; + let xBest = x; + let yBest = y; + for (const slot of targetPositions) { + const deltaX = slot.x - x; + const deltaY = slot.y - y; + const delta = Math.sqrt(deltaX * deltaX + deltaY * deltaY); + if (delta < deltaMin) { + deltaMin = delta; + xBest = slot.x; + yBest = slot.y; + } + } + if (deltaMin < 50) { + // review: how close do we want? + x = xBest; + y = yBest; + snapped = true; + } + dragTarget.style.top = y + "px"; + dragTarget.style.left = x + "px"; +}; + +const stopDrag = (e: PointerEvent) => { + // If they let go at a place that isn't a snap position at all, put it back where it was. + if (!snapped) { + const oldPosition = originalPositions.get(dragTarget); + dragTarget.style.top = oldPosition?.y + "px"; + dragTarget.style.left = oldPosition?.x + "px"; + } + dragTarget.removeEventListener("pointerup", stopDrag); + dragTarget.removeEventListener("pointermove", elementDrag); + + // If there was already a draggable in that slot, move the one we are replacing + // back to its original position. + // Enhance: animate? + const page = dragTarget.closest(".bloom-page") as HTMLElement; + const draggables = Array.from(page.querySelectorAll("[data-bubble-id]")); + draggables.forEach((elt: HTMLElement) => { + if (elt === dragTarget) { + return; + } + if ( + elt.offsetLeft === dragTarget.offsetLeft && + elt.offsetTop === dragTarget.offsetTop + ) { + const originalPosition = originalPositions.get(elt); + if (originalPosition) { + elt.style.left = originalPosition.x + "px"; + elt.style.top = originalPosition.y + "px"; + } + } + }); +}; + +const getVisibleText = (elt: HTMLElement): string => { + const visibleDivs = getVisibleEditables(elt); + return Array.from(visibleDivs) + .map((elt: HTMLElement) => elt.textContent) + .join(" "); +}; + +const rightPosition = ( + elt: HTMLElement, + correctX: number, + correctY: number +) => { + const actualX = elt.offsetLeft; + const actualY = elt.offsetTop; + return ( + // Since anything correct should be snapped, using a range probably isn't necessary + Math.abs(correctX - actualX) < 0.5 && Math.abs(correctY - actualY) < 0.5 + ); +}; + +export const performCheck = (e: MouseEvent) => { + const target = e.currentTarget as HTMLElement; + const page = target.closest(".bloom-page") as HTMLElement; + const allCorrect = checkDraggables(page) && checkRandomSentences(page); + + showCorrectOrWrongItems(page, allCorrect); + + return allCorrect; +}; + +export const performTryAgain = (e: MouseEvent) => { + const target = e.currentTarget as HTMLElement; + const page = target.closest(".bloom-page") as HTMLElement; + classSetter(page, "drag-activity-correct", false); + classSetter(page, "drag-activity-wrong", false); + //currently I don't think it could be set here, but make sure. + classSetter(page, "drag-activity-solution", false); +}; + +export const classSetter = ( + page: HTMLElement, + className: string, + wanted: boolean +) => { + if (wanted) { + page.parentElement?.classList.add(className); + } else { + page.parentElement?.classList.remove(className); + } +}; + +let draggableReposition: HTMLElement; +let wordBeingRepositioned: HTMLElement; +function showCorrectOrWrongItems(page: HTMLElement, correct: boolean) { + classSetter(page, "drag-activity-correct", correct); + classSetter(page, "drag-activity-wrong", !correct); + + // play sound + const soundFile = page.getAttribute( + correct ? "data-correct-sound" : "data-wrong-sound" + ); + const playOtherStuff = () => { + const elementsMadeVisible = Array.from( + page.getElementsByClassName( + correct ? "drag-item-correct" : "drag-item-wrong" + ) + ) as HTMLElement[]; + const possibleNarrationElements: HTMLElement[] = []; + const videoElements: HTMLVideoElement[] = []; + elementsMadeVisible.forEach(e => { + possibleNarrationElements.push(...getVisibleEditables(e)); + videoElements.push(...Array.from(e.getElementsByTagName("video"))); + }); + const playables = getAudioSentences(possibleNarrationElements); + playAllVideo(videoElements, () => playAllAudio(playables, page)); + }; + if (soundFile) { + const audio = new Audio(urlPrefix() + "/audio/" + soundFile); + audio.style.visibility = "hidden"; + // To my surprise, in BP storybook it works without adding the audio to any document. + // But in Bloom proper, it does not. I think it is because this code is part of the toolbox, + // so the audio element doesn't have the right context to interpret the relative URL. + page.append(audio); + // It feels cleaner if we remove it when done. This could fail, e.g., if the user + // switches tabs or pages before we get done playing. Removing it immediately + // prevents the sound being played. It's not a big deal if it doesn't get removed. + audio.play(); + audio.addEventListener( + "ended", + () => { + page.removeChild(audio); + playOtherStuff(); + }, + { once: true } + ); + } else { + playOtherStuff(); + } +} + +function checkDraggables(page: HTMLElement) { + let allCorrect = true; + const draggables = Array.from(page.querySelectorAll("[data-bubble-id]")); + draggables.forEach((draggableToCheck: HTMLElement) => { + const targetId = draggableToCheck.getAttribute("data-bubble-id"); + const target = page.querySelector( + `[data-target-of="${targetId}"]` + ) as HTMLElement; + if (!target) { + // this one is not required to be in a right place. + // Possibly we might one day need to check that it has NOT been dragged to a target. + // But for now, we only allow one draggable per target, so if this has been wrongly + // used some other one will not be in the right place. + return; + } + + const correctX = target.offsetLeft; + const correctY = target.offsetTop; + + if (!rightPosition(draggableToCheck, correctX, correctY)) { + // It's not in the expected place. But perhaps one with the same text is? + // This only applies if it's a text item. + // (don't use getElementsByClassName here...there could be a TG on an image description of + // a picture. To be a text item it must have a direct child that is a TG.) + if ( + !Array.from(draggableToCheck.children).some(x => + x.classList.contains("bloom-translationGroup") + ) + ) { + // not a text item. Two images or videos with the same (empty) text are not equivalent. + allCorrect = false; + return; + } + const visibleText = getVisibleText(draggableToCheck); + if ( + !draggables.some((otherDraggable: HTMLElement) => { + if (otherDraggable === draggableToCheck) { + return false; // already know this draggable is not at the right place + } + if (getVisibleText(otherDraggable) !== visibleText) { + return false; // only interested in ones with the same text + } + return rightPosition(otherDraggable, correctX, correctY); + }) + ) { + allCorrect = false; + } + } + }); + return allCorrect; +} + +let placeHolder: HTMLElement | undefined; +let startWidth = 0; +const draggableWordMargin = 5; // enhance: compute from element + +function startDragWordInSentence(e: PointerEvent) { + if (e.button !== 0) return; // only left button + if (e.ctrlKey) return; // ignore ctrl+click + + // get the pointer position etc. at startup: + wordBeingRepositioned = e.currentTarget as HTMLElement; + startWidth = wordBeingRepositioned.offsetWidth; // includes original padding but not margin + const page = wordBeingRepositioned.closest(".bloom-page") as HTMLElement; + const scale = getScale(page); + dragStartX = e.clientX / scale - wordBeingRepositioned.offsetLeft; + dragStartY = e.clientY / scale - wordBeingRepositioned.offsetTop; + + // Leave the original where it was and make a copy to drag around. + draggableReposition = wordBeingRepositioned.ownerDocument.createElement( + "div" + ); + wordBeingRepositioned.classList.forEach(c => + draggableReposition.classList.add(c) + ); + //draggableReposition.classList.add("drag-item-order-word"); + draggableReposition.textContent = wordBeingRepositioned.textContent; + draggableReposition.style.position = "absolute"; + draggableReposition.style.left = wordBeingRepositioned.offsetLeft + "px"; + draggableReposition.style.top = wordBeingRepositioned.offsetTop + "px"; + // We don't want it to show while we're dragging the clone. We need something to take up the space, + // though, until we decide it has moved. We could mess with its own properties, but then we have + // to put everything back. Also, we want to move it in the paragraph, and if we move the thing + // itself, we seem to lose our mouse capture. So we make a placeholder to take up the space. + placeHolder = makeAnimationPlaceholder(wordBeingRepositioned); + // don't add padding here, target still has it. Capture this before we hide it. + placeHolder.style.width = startWidth + draggableWordMargin + "px"; + wordBeingRepositioned.parentElement?.insertBefore( + placeHolder, + wordBeingRepositioned + ); + wordBeingRepositioned.style.display = "none"; + + // It's bizarre to put the listeners and pointer capture on the target, which is NOT being dragged, + // rather than the draggableReposition, which is. But it doesn't work to setPointerCapture on + // the draggableReposition. I think it's because the draggableReposition is not the object clicked. + // And once the mouse events are captured by the target, all mouse events go to that, so we get + // them properly while dragging, and can use them to move the draggableReposition. + wordBeingRepositioned.setPointerCapture(e.pointerId); + wordBeingRepositioned.addEventListener("pointerup", stopDragWordInSentence); + wordBeingRepositioned.addEventListener("pointermove", dragWordInSentence); + // not sure we need this. + // recommended by https://www.redblobgames.com/making-of/draggable/ to prevent touch movement + // dragging the page behind the draggable element. + wordBeingRepositioned.addEventListener("touchstart", preventTouchDefault); + wordBeingRepositioned.parentElement?.appendChild(draggableReposition); +} + +const preventTouchDefault = (e: TouchEvent) => { + e.preventDefault(); +}; + +let lastItemDraggedOver: HTMLElement | undefined; + +const dragWordInSentence = (e: PointerEvent) => { + const page = draggableReposition.closest(".bloom-page") as HTMLElement; + const scale = getScale(page); + e.preventDefault(); + const x = e.clientX / scale - dragStartX; + const y = e.clientY / scale - dragStartY; + + draggableReposition.style.top = y + "px"; + draggableReposition.style.left = x + "px"; + + if (animationInProgress) { + return; + } + const container = wordBeingRepositioned.parentElement!; + const itemDraggedOver = Array.from(container.children).find(c => { + const rect = c.getBoundingClientRect(); + return ( + c !== wordBeingRepositioned && + c !== placeHolder && + c !== draggableReposition && + e.clientX > rect.left && + e.clientX < rect.right && + e.clientY > rect.top && + e.clientY < rect.bottom + ); + }); + + // If we don't check for a different item, then when we drag a short word over a long one, the mouse + // may still be over the long word when the animation finishes, at which point it unhelpfully moves + // back. + if (itemDraggedOver && itemDraggedOver !== lastItemDraggedOver) { + const children = Array.from(container.children); + if ( + children.indexOf(itemDraggedOver) > children.indexOf(placeHolder!) + ) { + // moving right; it wants to go after the thing we dragged onto. + // (It's OK if nextSibling is null; gets inserted at end, which is what we want.) + animateMove(() => { + container.insertBefore( + placeHolder!, + itemDraggedOver.nextSibling + ); + }); + } else { + // moving left; it wants to go before the thing we dragged onto. + animateMove(() => { + container.insertBefore(placeHolder!, itemDraggedOver); + }); + } + } else { + // moved outside the sentence altogether. If we're below or to the right of the last item, + // move to the end. Enhance: should we move to the front if we're above or to the left? + const relatedItems = Array.from( + wordBeingRepositioned.parentElement!.getElementsByClassName( + "drag-item-order-word" + ) + ).filter( + x => + x !== wordBeingRepositioned && + x !== placeHolder && + x !== draggableReposition + ) as HTMLElement[]; + const lastItem = relatedItems[relatedItems.length - 1]; + const bounds = lastItem.getBoundingClientRect(); + if ( + e.clientY > bounds.bottom || + (e.clientX > bounds.right && e.clientY > bounds.top) + ) { + animateMove(() => { + container.appendChild(placeHolder!); + }); + } + } + lastItemDraggedOver = itemDraggedOver as HTMLElement; +}; + +const stopDragWordInSentence = (e: PointerEvent) => { + e.preventDefault(); + wordBeingRepositioned.style.visibility = "visible"; + wordBeingRepositioned.removeEventListener( + "pointerup", + stopDragWordInSentence + ); + wordBeingRepositioned.removeEventListener( + "pointermove", + dragWordInSentence + ); + wordBeingRepositioned.releasePointerCapture(e.pointerId); // redundant I think + wordBeingRepositioned.removeEventListener( + "touchstart", + preventTouchDefault + ); + // We're getting rid of this, so we don't need to remove the event handlers it has. + draggableReposition.parentElement?.removeChild(draggableReposition); + + wordBeingRepositioned.parentElement?.insertBefore( + wordBeingRepositioned, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + placeHolder! + ); + wordBeingRepositioned.parentElement?.removeChild(placeHolder!); + placeHolder = undefined; + wordBeingRepositioned.style.display = ""; // show it again +}; + +function makeAnimationPlaceholder(itemBeingRepositioned: HTMLElement) { + const placeholder = itemBeingRepositioned.cloneNode(true) as HTMLElement; + placeholder.style.overflowX = "hidden"; + placeholder.style.marginRight = "0"; // clear all these so it can shrink to taking up no space at all. + placeholder.style.paddingLeft = "0"; + placeholder.style.paddingRight = "0"; + placeholder.style.display = ""; // in case it was display:none + placeholder.style.visibility = "hidden"; //just takes up space for animation + return placeholder; +} + +let animationInProgress = false; + +function animateMove(movePlaceholder: () => void) { + animationInProgress = true; + const duration = 200; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const container = wordBeingRepositioned.parentElement!; + const duplicate = makeAnimationPlaceholder(wordBeingRepositioned); + container.insertBefore(duplicate, placeHolder!); + movePlaceholder(); + const start = Date.now(); + + const step = () => { + const elapsed = Date.now() - start; + const fraction = Math.min(elapsed / duration, 1); + // This width includes the original padding and margin, so that it takes up the original space + // to begin with, but can drop to zero. + const originalWordWidth = startWidth + draggableWordMargin; + if (!placeHolder) { + // terminated by mouseUp + container.removeChild(duplicate); + animationInProgress = false; + return; + } + placeHolder.style.width = originalWordWidth * fraction + "px"; + duplicate.style.width = originalWordWidth * (1 - fraction) + "px"; + if (fraction < 1) { + requestAnimationFrame(step); + } else { + // animation is over, clean up. + container.removeChild(duplicate); + placeHolder.style.width = originalWordWidth + "px"; // previous step might not have reached full size + animationInProgress = false; + } + }; + requestAnimationFrame(step); +} +function checkRandomSentences(page: HTMLElement) { + const sentences = page.getElementsByClassName("drag-item-random-sentence"); + for (let i = 0; i < sentences.length; i++) { + const sentence = sentences[i]; + // We check the expected text rather than the expected order of the child + // elements, because it automatically handles the possibility of repeated words. + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const correctAnswerWords = sentence + .getAttribute("data-answer")! + .split(" "); + const actualWordElements = Array.from(sentence.children); + for (let j = 0; j < actualWordElements.length; j++) { + const item = actualWordElements[j]; + if (item.textContent !== correctAnswerWords[j]) { + return false; + } + } + } + return true; +} + +export let draggingSlider = false; + +// Setup that is common to Play and design time +export function setupWordChooserSlider(page: HTMLElement) { + //Slider: const wrapper = page.getElementsByClassName( + // "bloom-activity-slider" + // )[0] as HTMLElement; + // if (!wrapper) { + // return; // panic? + // } + // wrapper.innerHTML = ""; // clear out any existing content. + // const slider = page.ownerDocument.createElement("div"); + // slider.classList.add("bloom-activity-slider-content"); + // slider.style.left = 0 + "px"; + // wrapper.appendChild(slider); + // dragStartX = 0; + // const scale = getScale(page); + // // Review: maybe we should use some sort of fancier slider? This one, for example, + // // won't have fancy effects like continuing to slide if you flick it. + // // But it's also possible this is good enough. Not really expecting a lot more items + // // than will fit. + // const moveHandler = (e: PointerEvent) => { + // let x = e.clientX / scale - dragStartX; + // if (Math.abs(x) > 4) { + // draggingSlider = true; + // } + // if (x > 0) { + // x = 0; + // } + // const maxScroll = Math.max(slider.offsetWidth - wrapper.offsetWidth, 0); + // if (x < -maxScroll) { + // x = -maxScroll; + // } + // slider.style.left = x + "px"; + // }; + // const upHandler = (e: PointerEvent) => { + // slider.removeEventListener("pointermove", moveHandler); + // page.ownerDocument.body.removeEventListener("pointerup", upHandler); + // setTimeout(() => { + // draggingSlider = false; + // }, 50); + // }; + // slider.addEventListener("pointerdown", e => { + // if (e.button !== 0) return; // only left button + // if (e.ctrlKey) return; // ignore ctrl+click + // dragStartX = e.clientX / scale - slider.offsetLeft; + // slider.addEventListener("pointermove", moveHandler); + // // We'd like to capture the pointer, and then we could put the up handler on the slider. + // // But then a click on an image inside the slider never gets the mouse up event, so never + // // gets a click. So we put the up handler on the body (so that it will get called even if + // // the up happens outside the slider). + // //slider.setPointerCapture(e.pointerId); + // page.ownerDocument.body.addEventListener("pointerup", upHandler); + // }); + // const imagesToPlace = shuffle( + // Array.from(page.querySelectorAll("[data-img-txt]")) + // ); + // imagesToPlace.forEach((imgTop: HTMLElement) => { + // const img = imgTop.getElementsByTagName("img")[0]; + // let sliderImgSrc = ""; + // if (img) { + // // An older comment said: + // // Not just img.src: that yields a full URL, which will show the image, but will not match + // // when we are later trying to find the corresponding original image. + // // I'm not finding anything that works that way, and the code below finds a full URL + // sliderImgSrc = img.getAttribute("src")!; + // } else { + // // In bloom-player, for a forgotten and possibly obsolete reason, we use a background image + // // on the container. (I vaguely recall it may be important when animating the main image.) + // const imgContainer = imgTop.getElementsByClassName( + // "bloom-imageContainer" + // )[0] as HTMLElement; + // if (!imgContainer) { + // return; // weird + // } + // const bgImg = imgContainer.style.backgroundImage; + // if (!bgImg) { + // return; // weird + // } + // const start = bgImg.indexOf('"'); + // const end = bgImg.lastIndexOf('"'); + // sliderImgSrc = bgImg.substring(start + 1, end); + // } + // // not using cloneNode here because I don't want to bring along any alt text that might provide a clue + // const sliderImg = imgTop.ownerDocument.createElement("img"); + // sliderImg.src = sliderImgSrc; + // sliderImg.ondragstart = () => false; + // sliderImg.setAttribute( + // "data-img", + // imgTop.getAttribute("data-img-txt")! + // ); + // const sliderItem = imgTop.ownerDocument.createElement("div"); + // sliderItem.classList.add("bloom-activity-slider-item"); + // sliderItem.appendChild(sliderImg); + // slider.appendChild(sliderItem); + // }); + // if (slider.offsetWidth > wrapper.offsetWidth) { + // // We need a slider effect. We want one of the images to be partly visible as a clue that + // // sliding is possible. + // const avWidth = slider.offsetWidth / imagesToPlace.length; + // let indexNearBorder = Math.floor(wrapper.offsetWidth / avWidth); + // let sliderItem = slider.children[indexNearBorder] as HTMLElement; + // if (sliderItem.offsetLeft > wrapper.offsetWidth - 30) { + // // The item we initially selected is mostly off the right edge. + // // Stretch things to make the previous item half-off-screen. + // indexNearBorder--; + // sliderItem = slider.children[indexNearBorder] as HTMLElement; + // } + // if ( + // sliderItem.offsetLeft + sliderItem.offsetWidth < + // wrapper.offsetWidth + 30 + // ) { + // const oldMarginPx = + // sliderItem.ownerDocument.defaultView?.getComputedStyle( + // sliderItem + // ).marginLeft ?? "22px"; + // const oldMargin = parseInt( + // oldMarginPx.substring(0, oldMarginPx.length - 2) + // ); + // const desiredLeft = + // wrapper.offsetWidth - sliderItem.offsetWidth / 2; + // const newMargin = + // oldMargin + + // (desiredLeft - sliderItem.offsetLeft) / indexNearBorder / 2; + // Array.from(slider.children).forEach((elt: HTMLElement) => { + // elt.style.marginLeft = newMargin + "px"; + // elt.style.marginRight = newMargin + "px"; + // }); + // } + // } +} + +//Slider: const clickSliderImage = (e: MouseEvent) => { +// if (draggingSlider) { +// return; +// } +// const img = e.currentTarget as HTMLElement; +// const page = img.closest(".bloom-page") as HTMLElement; +// const activeTextBox = page.getElementsByClassName("bloom-activeTextBox")[0]; +// if (!activeTextBox) { +// return; // weird +// } +// var activeId = activeTextBox.getAttribute("data-txt-img"); +// const imgId = img.getAttribute("data-img"); +// if (activeId === imgId) { +// const imgTop = page.querySelector(`[data-img-txt="${imgId}"]`); +// if (!imgTop) { +// return; // weird +// } +// imgTop.classList.remove("bloom-hideSliderImage"); +// setTimeout(() => { +// if (!showARandomWord(page, true)) { +// showCorrectOrWrongItems(page, true); +// } +// }, 1000); // should roughly correspond to the css transition showing the item +// } else { +// showCorrectOrWrongItems(page, false); +// } +// }; + +// function setupSliderImageEvents(page: HTMLElement) { +// const slider = page.getElementsByClassName("bloom-activity-slider")[0]; +// if (!slider) { +// return; // panic? +// } +// const sliderImages = Array.from(slider.getElementsByTagName("img")); +// sliderImages.forEach((img: HTMLElement) => { +// img.addEventListener("click", clickSliderImage); +// }); +// } + +// export function setSlideablesVisibility(page: HTMLElement, visible: boolean) { +// const slideables = Array.from(page.querySelectorAll("[data-img-txt]")); +// slideables.forEach((elt: HTMLElement) => { +// if (visible) { +// elt.classList.remove("bloom-hideSliderImage"); +// } else { +// elt.classList.add("bloom-hideSliderImage"); +// } +// }); +// } + +// function showARandomWord(page: HTMLElement, playAudio: boolean) { +// const possibleWords = Array.from(page.querySelectorAll("[data-txt-img]")); +// const targetWords = possibleWords.filter(w => { +// const imgId = w.getAttribute("data-txt-img"); +// const img = page.querySelector(`[data-img-txt="${imgId}"]`); +// return img?.classList.contains("bloom-hideSliderImage"); +// }); +// possibleWords.forEach(w => { +// w.classList.remove("bloom-activeTextBox"); +// }); +// if (targetWords.length === 0) { +// return false; +// } + +// const randomIndex = Math.floor(Math.random() * targetWords.length); +// targetWords[randomIndex].classList.add("bloom-activeTextBox"); +// if (playAudio) { +// const playables = getAudioSentences([ +// targetWords[randomIndex] as HTMLElement +// ]); +// playAllAudio(playables); +// } +// return true; +// } diff --git a/src/event.ts b/src/event.ts index ddd404e4..2cfea562 100644 --- a/src/event.ts +++ b/src/event.ts @@ -1,15 +1,17 @@ // originally from http://stackoverflow.com/a/14657922/723299 interface ILiteEvent { - subscribe(handler: (data?: T)=> void): void; + subscribe(handler: (data?: T) => void): void; unsubscribe(handler: (data?: T) => void): void; } export default class LiteEvent implements ILiteEvent { - private handlers: Array<((data?: T) => void)> = []; + private handlers: Array<(data?: T) => void> = []; public subscribe(handler: (data?: T) => void) { - this.handlers.push(handler); + if (this.handlers.indexOf(handler) === -1) { + this.handlers.push(handler); + } } public unsubscribe(handler: (data?: T) => void) { diff --git a/src/music.ts b/src/music.ts index 1c2deebe..00b23d32 100644 --- a/src/music.ts +++ b/src/music.ts @@ -1,10 +1,11 @@ import LiteEvent from "./event"; -import { BloomPlayerCore, PlaybackMode } from "./bloom-player-core"; +import { BloomPlayerCore } from "./bloom-player-core"; import { logSound, logSoundPaused, logSoundRepeat } from "./videoRecordingSupport"; +import { PlaybackMode, currentPlaybackMode } from "./narration"; interface ISelection { id: number; @@ -48,7 +49,7 @@ export class Music { private listen() { this.setMusicSourceAndVolume(); - if (BloomPlayerCore.currentPlaybackMode === PlaybackMode.AudioPaused) { + if (currentPlaybackMode === PlaybackMode.AudioPaused) { this.getPlayer().pause(); } else { this.playerPlay(); diff --git a/src/narration.test.ts b/src/narration.test.ts index 818c4881..04bcdef6 100644 --- a/src/narration.test.ts +++ b/src/narration.test.ts @@ -1,4 +1,4 @@ -import { sortAudioElements } from "./narrationUtils"; +import { sortAudioElements } from "./narration"; import { createDiv, createPara, createSpan } from "./test/testHelper"; test("sortAudioElements with no tabindexes preserves order", () => { diff --git a/src/narration.ts b/src/narration.ts index 0f41c231..677ffd83 100644 --- a/src/narration.ts +++ b/src/narration.ts @@ -1,8 +1,149 @@ +// This file contains code for playing audio in a bloom page, including a draggable page. +// The file is designed to be shared between Bloom Desktop and Bloom Player. Maybe eventually +// as an npm module, but for now, just copied into both projects. See comments in dragActivityRuntime.ts. + +// It is quite difficult to know how to handle audio in a drag activity page. +// We need to be able to play it both during "Play" and when showing the page in BP. +// The code in this file so far represents a reuniting of the original Bloom Player +// narration code with the code developed for narration in drag activities. +// There is existing code for playing audio in Bloom Desktop, but it is entangled with the +// code that manages audio recording and the talking book tool. I hope eventually +// this code, or parts of it, can be used there as well as in 'Play' mode. +// The code here has hooks and some methods which are only useful to BP, for things such as +// autoplay and page advance, and also cases where the text being played is +// not fully visible. It deals with pause and continue, which also interact with +// the Bloom Player controls and with video and background music. +// Drag activities added the need to handle situations where the audio on the page +// should not all be played in succession, and more complicated sequencing of audio +// and video + import LiteEvent from "./event"; -import { BloomPlayerCore, PlaybackMode } from "./bloom-player-core"; -import { sortAudioElements, ISetHighlightParams } from "./narrationUtils"; -import { SwiperInstance } from "react-id-swiper"; -import { logSound } from "./videoRecordingSupport"; +// Note: trying to avoid other imports, as part of the process of moving this code to a module +// that can be shared with BloomDesktop. + +//----This first block is the old narrationUtils.ts. Everything here is now meant to be reusable +// code connected with narration, not specific to Bloom Player. +export interface ISetHighlightParams { + newElement: Element; + shouldScrollToElement: boolean; + disableHighlightIfNoAudio?: boolean; + oldElement?: Element | null | undefined; // Optional. Provides some minor optimization if set. +} + +export enum PlaybackMode { + NewPage, // starting a new page ready to play + NewPageMediaPaused, // starting a new page in the "paused" state + VideoPlaying, // video is playing + VideoPaused, // video is paused + AudioPlaying, // narration and/or animation are playing (or possibly finished) + AudioPaused, // narration and/or animation are paused + MediaFinished // video, narration, and/or animation has played (possibly no media to play) + // Note that music can be playing when the state is either AudioPlaying or MediaFinished. +} + +export let currentPlaybackMode: PlaybackMode = PlaybackMode.NewPage; +export function setCurrentPlaybackMode(mode: PlaybackMode) { + currentPlaybackMode = mode; +} + +// These functions support allowing a client (typically BloomPlayerCore) to register as +// the object that wants to receive notification of how long audio was played. +// Duration is in seconds. +export let durationReporter: (duration: number) => void; +export function listenForPlayDuration(reporter: (duration: number) => void) { + durationReporter = reporter; +} + +// A client may configure a function which can be called to find out whether a swipe +// is in progress...in BloomPlayerCore, this is implemented by a test on SWiper. +// It is currently only used when we need to scroll the content of a field we are +// playing. Bloom Desktop does not need to set it. +export let isSwipeInProgress: () => boolean; +export function setTestIsSwipeInProgress(tester: () => boolean) { + isSwipeInProgress = tester; +} + +// A client may configure a function which is passed the URL of each audio file narration plays. +export let logNarration: (src: string) => void = () => { + // do nothing by default +}; +export function setLogNarration(logger: (src: string) => void) { + logNarration = + logger ?? + (() => { + // do nothing by default (so we don't have to check for null) + }); +} + +let playerUrlPrefix = ""; +// In bloom player, figuring the url prefix (to put before file JS needs to locate, like +// sounds to play) is complicated. We pass it in. +// In Bloom desktop, it's almost more tricky. If we're executing in the page iframe, we can +// easily derive it from the page's URL. But if we're executing in the toolbox, that doesn't work. +// So we arrange for newPageReady, called at least as often as changing book, to set it up +// using setPlayerUrlPrefixFromWindowLocationHref +export function setPlayerUrlPrefix(prefix: string) { + playerUrlPrefix = prefix; +} + +export function setPlayerUrlPrefixFromWindowLocationHref(bookSrc: string) { + setPlayerUrlPrefix(getUrlPrefixFromWindowHref(bookSrc)); +} + +function getUrlPrefixFromWindowHref(bookSrc: string) { + const index = bookSrc.lastIndexOf("/"); + return bookSrc.substring(0, index); +} + +export function urlPrefix(): string { + if (playerUrlPrefix) { + return playerUrlPrefix; + } + return getUrlPrefixFromWindowHref(window.location.href); +} + +// We need to sort these by the tabindex of the containing bloom-translationGroup element. +// We need a stable sort, which array.sort() does not provide: elements in the same +// bloom-translationGroup, or where the translationGroup does not have a tabindex, +// should remain in their current order. +// It's not obvious what should happen to TGs with no tabindex when others have it. +// At this point we're going with the approach that no tabindex is equivalent to tabindex 999. +// This should cause text with no tabindex to sort to the bottom, if other text has a tabindex; +// It should also not affect order in situations where no text has a tabindex +// (An earlier algorithm attempted to preserve document order for the no-tab-index case +// by comparing any two elements using document order if either lacks tabindex. +// This works well for many cases, but if there's a no-tabindex element between two +// that get re-ordered (e.g., ABCDEF where the only tabindexes are C=2 and E=1), +// the function is not transitive (e.g. C < D < E but E < C) which will produce +// unpredictable results. +export function sortAudioElements(input: HTMLElement[]): HTMLElement[] { + const keyedItems = input.map((item, index) => { + return { tabindex: getTgTabIndex(item), index, item }; + }); + keyedItems.sort((x, y) => { + // If either is not in a translation group with a tabindex, + // order is determined by their original index. + // Likewise if the tabindexes are the same. + if (!x.tabindex || !y.tabindex || x.tabindex === y.tabindex) { + return x.index - y.index; + } + // Otherwise, determined by the numerical order of tab indexes. + return parseInt(x.tabindex, 10) - parseInt(y.tabindex, 10); + }); + return keyedItems.map(x => x.item); +} + +function getTgTabIndex(input: HTMLElement): string | null { + let tg: HTMLElement | null = input; + while (tg && !tg.classList.contains("bloom-translationGroup")) { + tg = tg.parentElement; + } + if (!tg) { + return "999"; + } + return tg.getAttribute("tabindex") || "999"; +} +///---- end of the bit that ended up in narrationUtils.ts before the merge. const kSegmentClass = "bloom-highlightSegment"; @@ -13,1089 +154,1081 @@ const kEnableHighlightClass = "ui-enableHighlight"; // For example, some elements have highlighting prevented at this level // because its content has been broken into child elements, only some of which show the highlight const kDisableHighlightClass = "ui-disableHighlight"; - // Indicates that highlighting is briefly/temporarily suppressed, // but may become highlighted later. // For example, audio highlighting is suppressed until the related audio starts playing (to avoid flashes) const kSuppressHighlightClass = "ui-suppressHighlight"; -var durationOfPagesWithoutNarration = 3.0; // seconds +let durationOfPagesWithoutNarration = 3.0; // seconds export function setDurationOfPagesWithoutNarration(d: number) { durationOfPagesWithoutNarration = d; } // Even though these can now encompass more than strict sentences, // we continue to use this class name for backwards compatability reasons. -const kAudioSentence = "audio-sentence"; +export const kAudioSentence = "audio-sentence"; const kImageDescriptionClass = "bloom-imageDescription"; -// Handles implementation of narration, including playing the audio and -// highlighting the currently playing text. -// Enhance: There's code here to support PageNarrationComplete for auto-advance, -// but that isn't implemented yet so it may not be complete. -// May need to copy more pieces from old BloomPlayer. -// Enhance: Pause is a prop for this control, but somehow we need to -// notify the container if we are paused forcibly by Chrome refusing to -// let us play until the user interacts with the page. -export default class Narration { - public playerPage: HTMLElement; - public swiperInstance: SwiperInstance | null = null; - public urlPrefix: string; - // The time we started to play the current page (set in computeDuration, adjusted for pauses) - private startPlay: Date; - private startPause: Date; - private fakeNarrationAborted: boolean = false; - private fakeNarrationTimer: number; - public pageNarrationCompleteTimer: number; - private segmentIndex: number; - - private segments: HTMLElement[]; - - private currentAudioId: string; - - // The first one to play should be at the end for all of these - private elementsToPlayConsecutivelyStack: HTMLElement[] = []; // The audio-sentence elements (ie those with actual audio files associated with them) that should play one after the other - private subElementsWithTimings: Array<[Element, number]> = []; - - // delay responding if there is no narration to play (to give animation fixed time to display) - public PageNarrationComplete: LiteEvent; - public PageDurationAvailable: LiteEvent; - // respond immediately if there is no narration to play. - public PlayCompleted: LiteEvent; - public PlayFailed: LiteEvent; - public PageDuration: number; - - // We want Narration to inform its controllers when we start/stop reading - // image descriptions. - public ToggleImageDescription: LiteEvent; - - // A Session Number that keeps track of each time playAllSentences started. - // This is used to determine whether the page has been changed or not. - private currentAudioSessionNum: number = 0; - - private includeImageDescriptions: boolean = true; - - // This represents the start time of the current playing of the audio. If the user presses pause/play, it will be reset. - // This is used for analytics reporting purposes - private audioPlayCurrentStartTime: number | null = null; // milliseconds (since 1970/01/01, from new Date().getTime()) - - public setSwiper(newSwiperInstance: SwiperInstance | null) { - this.swiperInstance = newSwiperInstance; +// The page that is currently being played (or edited, when we use this code in Bloom Desktop). +let currentPlayPage: HTMLElement | null = null; +// When we have recently changed pages, stores a value returned from setTimeout, which can be used to cancel the old timeout +// if we change pages again. After three seconds, the timerout sets it to zero. A non-zero value also prevents the code that +// tries to scroll the currently-playing text into view from doing so in the early stages of viewing a page, when it +// can cause problems for swiper. +let recentPageChange: any = 0; // any because typescript thinks we're in Nodejs and setTimeout will return an object. +// Unused in Bloom desktop, but in Bloom player, current page might change while a series of sounds +// is playing. This lets us avoid starting the next sound if the page has changed in the meantime. +export function setCurrentPage(page: HTMLElement) { + if (page === currentPlayPage) return; + if (recentPageChange) { + clearTimeout(recentPageChange); } + recentPageChange = setTimeout(() => { + recentPageChange = 0; + }, 3000); + currentPlayPage = page; +} +export function getCurrentPage(): HTMLElement { + return currentPlayPage!; +} - // Roughly equivalent to BloomDesktop's AudioRecording::listen() function. - // As long as there is audio on the page, this method will play it. - public playAllSentences(page: HTMLElement | null): void { - if (!page && !this.playerPage) { - return; // this shouldn't happen - } - if (page) { - this.playerPage = page; - } - const mediaPlayer = this.getPlayer(); - if (mediaPlayer) { - mediaPlayer.pause(); - mediaPlayer.currentTime = 0; - } +// The time we started playing the current narration. If we pause and resume this is adjusted +// by the length of the pause, so that "now" minus startPlay is always how much of the sound +// has actually been played. +let startPlay: Date; + +// When the most recent pause happened. +let startPause: Date; + +// Timer used to raise PageNarrationComplete after a delay when there is no audio on the page. +// Gets canceled if we pause and restarted if we resume. +let fakeNarrationTimer: number; + +let segments: HTMLElement[]; + +let currentAudioId = ""; + +// The first one to play should be at the end for both of these +let elementsToPlayConsecutivelyStack: HTMLElement[] = []; +let subElementsWithTimings: Array<[Element, number]> = []; + +// On a typical page with narration, these are raised at the same time, when the last narration +// on the page finishes. But if there is no narration at all, PlayCompleted will be raised +// immediately (useful for example to disable a pause button), but PageNarrationComplete will +// be raised only after the standard delay for non-audio page (useful for auto-advancing to the next page). +export const PageNarrationComplete = new LiteEvent(); +export const PlayCompleted = new LiteEvent(); +// Raised when we can't play narration, specifically because the browser won't allow it until +// the user has interacted with the page. +export const PlayFailed = new LiteEvent(); + +// This event allows Narration to inform its controllers when we start/stop reading +// image descriptions. It is raised for each segment we read and passed true if the one +// we are about to read is an image description, false otherwise. +// Todo: wants a better name, it's not about toggling whether something is an image description, +// but about possibly updating the UI to reflect whether we are reading one. +export const ToggleImageDescription = new LiteEvent(); + +// A Session Number that keeps track of each time playAllAudio started. +// This might be needed to keep track of changing pages, or when we start new audio +// that will replace something already playing. +let currentAudioSessionNum: number = 0; + +let includeImageDescriptions: boolean = true; +export function setIncludeImageDescriptions(b: boolean) { + includeImageDescriptions = b; +} - // Invalidate old ID, even if there's no new audio to play. - // (Deals with the case where you are on a page with audio, switch to a page without audio, then switch back to original page) - ++this.currentAudioSessionNum; - - this.fixHighlighting(); - - // Sorted into the order we want to play them, then reversed so we - // can more conveniently pop the next one to play from the end of the stack. - this.elementsToPlayConsecutivelyStack = sortAudioElements( - this.getPageAudioElements() - ).reverse(); - - const stackSize = this.elementsToPlayConsecutivelyStack.length; - if (stackSize === 0) { - // Nothing to play. Wait the standard amount of time anyway, in case we're autoadvancing. - if (this.PageNarrationComplete) { - this.pageNarrationCompleteTimer = window.setTimeout(() => { - this.PageNarrationComplete.raise(); - }, durationOfPagesWithoutNarration * 1000); - } - if (this.PlayCompleted) { - this.PlayCompleted.raise(); - } - return; - } +// This represents the start time of the current playing of the audio. If the user presses pause/play, it will be reset. +// This is used for analytics reporting purposes +let audioPlayCurrentStartTime: number | null = null; // milliseconds (since 1970/01/01, from new Date().getTime()) + +// Roughly equivalent to BloomDesktop's AudioRecording::listen() function. +// As long as there is audio on the page, this method will play it. +export function playAllSentences(page: HTMLElement | null): void { + if (!page && !currentPlayPage) { + return; // this shouldn't happen + } + const pageToPlay = page ?? currentPlayPage!; + playAllAudio(getPageAudioElements(pageToPlay), pageToPlay); +} + +export function playAllAudio(elements: HTMLElement[], page: HTMLElement): void { + setCurrentPage(page); + segments = getPageAudioElements(page); + startPlay = new Date(); + const mediaPlayer = getPlayer(); + if (mediaPlayer) { + // This felt like a good idea. But we are about to set a new src on the media player and play that, + // which will deal with any sound that is still playing. + // And if we explicitly pause it now, that actually starts an async process of getting it paused, which + // may not have completed by the time we attempt to play the new audio. And then play() fails. + //mediaPlayer.pause(); + mediaPlayer.currentTime = 0; + } + + // Invalidate old ID, even if there's no new audio to play. + // (Deals with the case where you are on a page with audio, switch to a page without audio, then switch back to original page) + ++currentAudioSessionNum; + + fixHighlighting(elements); - const firstElementToPlay = this.elementsToPlayConsecutivelyStack[ - stackSize - 1 - ]; // Remember to pop it when you're done playing it. (i.e., in playEnded) + // Sorted into the order we want to play them, then reversed so we + // can more conveniently pop the next one to play from the end of the stack. + elementsToPlayConsecutivelyStack = sortAudioElements(elements).reverse(); - this.setSoundAndHighlight(firstElementToPlay, true); - this.playCurrentInternal(); + const stackSize = elementsToPlayConsecutivelyStack.length; + if (stackSize === 0) { + // Nothing to play. First, raise the event that indicates nothing is playing. + // It typically sets mode to MediaFinsished, and we want to override that. + if (PlayCompleted) { + PlayCompleted?.raise(page); + } + // Simulate playing for a fixed amount of time before raising PageNarrationComplete, in case we're autoadvancing. + // We're not really playing, but we're pretending, so things work best if we go to that mode. + // For example, if we leave it in MediaFinished from the previous page or from raising PlayCompleted, pause won't work. + setCurrentPlaybackMode(PlaybackMode.AudioPlaying); + if (PageNarrationComplete) { + fakeNarrationTimer = window.setTimeout(() => { + setCurrentPlaybackMode(PlaybackMode.MediaFinished); + PageNarrationComplete?.raise(page); + }, durationOfPagesWithoutNarration * 1000); + } return; } - // Match space or   (\u00a0) or ​ (\u200b). Must have three or more in a row to match. - // Geckofx would typically give something like `    ` but wv2 usually gives something like `    ` - private multiSpaceRegex = /[ \u00a0\u200b]{3,}/; - private multiSpaceRegexGlobal = new RegExp(this.multiSpaceRegex, "g"); - - /** - * Finds and fixes any elements on the page that should have their audio-highlighting disabled. - * - * Note, all this logic is essentially duplicated from BloomDesktop where there are quite a few unit tests. - */ - private fixHighlighting() { - // Note: Only relevant when playing by sentence (but note, this can make Record by Text Box -> Split or Record by Sentence, Play by Sentence) - // Play by Text Box highlights the whole paragraph and none of this really matters. - // (the span selector won't match anyway) - const audioElements = this.getPageAudioElements(); - audioElements.forEach(audioElement => { - // FYI, don't need to process the bloom-linebreak spans. Nothing bad happens, just unnecessary. - const matches = this.findAll( - "span[id]:not(.bloom-linebreak)", - audioElement, - true + const firstElementToPlay = elementsToPlayConsecutivelyStack[stackSize - 1]; // Remember to pop it when you're done playing it. (i.e., in playEnded) + // I didn't comment this at the time, but my recollection is that making a new player + // each time helps with some cases where the old one was in a bad state, + // such as in the middle of pausing. Don't do this between setting highlight and playing, though, + // or the handler that removes the highlight suppression will be lost. + const src = mediaPlayer.getAttribute("src") ?? ""; + mediaPlayer.remove(); + + setSoundAndHighlight(firstElementToPlay, true); + // Review: do we need to do something to let the rest of the world know about this? + setCurrentPlaybackMode(PlaybackMode.AudioPlaying); + playCurrentInternal(); + return; +} + +// Match space or   (\u00a0) or ​ (\u200b). Must have three or more in a row to match. +// Geckofx would typically give something like `    ` but wv2 usually gives something like `    ` +const multiSpaceRegex = /[ \u00a0\u200b]{3,}/; +const multiSpaceRegexGlobal = new RegExp(multiSpaceRegex, "g"); +/** + * Finds and fixes any elements on the page that should have their audio-highlighting disabled. + * + * Note, all this logic is essentially duplicated from BloomDesktop where there are quite a few unit tests. + */ +function fixHighlighting(audioElements: HTMLElement[]) { + // Note: Only relevant when playing by sentence (but note, this can make Record by Text Box -> Split or Record by Sentence, Play by Sentence) + // Play by Text Box highlights the whole paragraph and none of this really matters. + // (the span selector won't match anyway) + audioElements.forEach(audioElement => { + // FYI, don't need to process the bloom-linebreak spans. Nothing bad happens, just unnecessary. + const matches = findAll( + "span[id]:not(.bloom-linebreak)", + audioElement, + true + ); + matches.forEach(element => { + // Remove all existing highlight classes from element and element's descendants. + // These shouldn't be in the dom as the editor is supposed to clean them up, + // but we have seen at least on case where it didn't. BL-13428. + removeHighlightClasses(element); + + // Simple check to help ensure that elements that don't need to be modified will remain untouched. + // This doesn't consider whether text that shouldn't be highlighted is already in inside an + // element with highlight disabled, but that's ok. The code down the stack checks that. + const containsNonHighlightText = !!element.innerText.match( + multiSpaceRegex ); - matches.forEach(element => { - // Remove all existing highlight classes from element and element's descendants. - // These shouldn't be in the dom as the editor is supposed to clean them up, - // but we have seen at least on case where it didn't. BL-13428. - this.removeHighlightClasses(element); - - // Simple check to help ensure that elements that don't need to be modified will remain untouched. - // This doesn't consider whether text that shouldn't be highlighted is already in inside an - // element with highlight disabled, but that's ok. The code down the stack checks that. - const containsNonHighlightText = !!element.innerText.match( - this.multiSpaceRegex - ); - if (containsNonHighlightText) { - this.fixHighlightingInNode(element, element); - } - }); + if (containsNonHighlightText) { + fixHighlightingInNode(element, element); + } }); - } + }); +} - // Remove all existing highlight classes from element and element's descendants. - private removeHighlightClasses(element: HTMLElement) { - element.classList.remove(kDisableHighlightClass); - element.classList.remove(kEnableHighlightClass); +// Remove all existing highlight classes from element and element's descendants. +function removeHighlightClasses(element: HTMLElement) { + element.classList.remove(kDisableHighlightClass); + element.classList.remove(kEnableHighlightClass); - Array.from(element.children).forEach((child: HTMLElement) => { - this.removeHighlightClasses(child); + Array.from(element.children).forEach((child: HTMLElement) => { + removeHighlightClasses(child); + }); +} + +/** + * Recursively fixes the audio-highlighting within a node (whether element node or text node) + * @param node The node to recursively fix + * @param startingSpan The starting span, AKA the one that will receive .ui-audioCurrent in the future. + */ +function fixHighlightingInNode(node: Node, startingSpan: HTMLSpanElement) { + if ( + node.nodeType === Node.ELEMENT_NODE && + (node as Element).classList.contains(kDisableHighlightClass) + ) { + // No need to process bloom-highlightDisabled elements (they've already been processed) + return; + } else if (node.nodeType === Node.TEXT_NODE) { + // Leaf node. Fix the highlighting, then go back up the stack. + fixHighlightingInTextNode(node, startingSpan); + return; + } else { + // Recursive case + const childNodesCopy = Array.from(node.childNodes); // Make a copy because node.childNodes is being mutated + childNodesCopy.forEach(childNode => { + fixHighlightingInNode(childNode, startingSpan); }); } +} - /** - * Recursively fixes the audio-highlighting within a node (whether element node or text node) - * @param node The node to recursively fix - * @param startingSpan The starting span, AKA the one that will receive .ui-audioCurrent in the future. - */ - private fixHighlightingInNode(node: Node, startingSpan: HTMLSpanElement) { - if ( - node.nodeType === Node.ELEMENT_NODE && - (node as Element).classList.contains(kDisableHighlightClass) - ) { - // No need to process bloom-highlightDisabled elements (they've already been processed) - return; - } else if (node.nodeType === Node.TEXT_NODE) { - // Leaf node. Fix the highlighting, then go back up the stack. - this.fixHighlightingInTextNode(node, startingSpan); - return; - } else { - // Recursive case - const childNodesCopy = Array.from(node.childNodes); // Make a copy because node.childNodes is being mutated - childNodesCopy.forEach(childNode => { - this.fixHighlightingInNode(childNode, startingSpan); - }); - } +/** + * Analyzes a text node and fixes its highlighting. + */ +function fixHighlightingInTextNode( + textNode: Node, + startingSpan: HTMLSpanElement +) { + if (textNode.nodeType !== Node.TEXT_NODE) { + throw new Error( + "Invalid argument to fixMultiSpaceInTextNode: node must be a TextNode" + ); } - /** - * Analyzes a text node and fixes its highlighting. - */ - private fixHighlightingInTextNode( - textNode: Node, - startingSpan: HTMLSpanElement - ) { - if (textNode.nodeType !== Node.TEXT_NODE) { - throw new Error( - "Invalid argument to fixMultiSpaceInTextNode: node must be a TextNode" - ); - } - - if (!textNode.nodeValue) { - return; - } + if (!textNode.nodeValue) { + return; + } - // string.matchAll would be cleaner, but not supported in all browsers (in particular, FF60) - // Use RegExp.exec for greater compatibility. - this.multiSpaceRegexGlobal.lastIndex = 0; // RegExp.exec is stateful! Need to reset the state. - const matches: { - text: string; - startIndex: number; - endIndex: number; // the index of the first character to exclude - }[] = []; - let regexResult: RegExpExecArray | null; - while ( - (regexResult = this.multiSpaceRegexGlobal.exec( - textNode.nodeValue - )) != null - ) { - regexResult.forEach(matchingText => { - matches.push({ - text: matchingText, - startIndex: - this.multiSpaceRegexGlobal.lastIndex - - matchingText.length, - endIndex: this.multiSpaceRegexGlobal.lastIndex // the index of the first character to exclude - }); + // string.matchAll would be cleaner, but not supported in all browsers (in particular, FF60) + // Use RegExp.exec for greater compatibility. + multiSpaceRegexGlobal.lastIndex = 0; // RegExp.exec is stateful! Need to reset the state. + const matches: { + text: string; + startIndex: number; + endIndex: number; // the index of the first character to exclude + }[] = []; + let regexResult: RegExpExecArray | null; + while ( + (regexResult = multiSpaceRegexGlobal.exec(textNode.nodeValue)) != null + ) { + regexResult.forEach(matchingText => { + matches.push({ + text: matchingText, + startIndex: + multiSpaceRegexGlobal.lastIndex - matchingText.length, + endIndex: multiSpaceRegexGlobal.lastIndex // the index of the first character to exclude }); - } + }); + } - // First, generate the new DOM elements with the fixed highlighting. - const newNodes: Node[] = []; - if (matches.length === 0) { - // No matches - newNodes.push(this.makeHighlightedSpan(textNode.nodeValue)); - } else { - let lastMatchEndIndex = 0; // the index of the first character to exclude of the last match - for (let i = 0; i < matches.length; ++i) { - const match = matches[i]; + // First, generate the new DOM elements with the fixed highlighting. + const newNodes: Node[] = []; + if (matches.length === 0) { + // No matches + newNodes.push(makeHighlightedSpan(textNode.nodeValue)); + } else { + let lastMatchEndIndex = 0; // the index of the first character to exclude of the last match + for (let i = 0; i < matches.length; ++i) { + const match = matches[i]; + + const preMatchText = textNode.nodeValue.slice( + lastMatchEndIndex, + match.startIndex + ); + lastMatchEndIndex = match.endIndex; + if (preMatchText) newNodes.push(makeHighlightedSpan(preMatchText)); - const preMatchText = textNode.nodeValue.slice( - lastMatchEndIndex, - match.startIndex - ); - lastMatchEndIndex = match.endIndex; - if (preMatchText) - newNodes.push(this.makeHighlightedSpan(preMatchText)); - - newNodes.push(document.createTextNode(match.text)); - - if (i === matches.length - 1) { - const postMatchText = textNode.nodeValue.slice( - match.endIndex - ); - if (postMatchText) { - newNodes.push(this.makeHighlightedSpan(postMatchText)); - } + newNodes.push(document.createTextNode(match.text)); + + if (i === matches.length - 1) { + const postMatchText = textNode.nodeValue.slice(match.endIndex); + if (postMatchText) { + newNodes.push(makeHighlightedSpan(postMatchText)); } } } + } - // Next, replace the old DOM element with the new DOM elements - const oldNode = textNode; - if (oldNode.parentNode && newNodes && newNodes.length > 0) { - for (let i = 0; i < newNodes.length; ++i) { - const nodeToInsert = newNodes[i]; - oldNode.parentNode.insertBefore(nodeToInsert, oldNode); - } + // Next, replace the old DOM element with the new DOM elements + const oldNode = textNode; + if (oldNode.parentNode && newNodes && newNodes.length > 0) { + for (let i = 0; i < newNodes.length; ++i) { + const nodeToInsert = newNodes[i]; + oldNode.parentNode.insertBefore(nodeToInsert, oldNode); + } - oldNode.parentNode.removeChild(oldNode); + oldNode.parentNode.removeChild(oldNode); - // We need to set ancestor's background back to transparent (instead of highlighted), - // and let each of the newNodes's styles control whether to be highlighted or transparent. - // If ancestor was highlighted but one of its new descendant nodes was transparent, - // all that would happen is the descendant would allow the ancestor's highlight color to show through, - // which doesn't achieve what we want :( - startingSpan.classList.add(kDisableHighlightClass); - } + // We need to set ancestor's background back to transparent (instead of highlighted), + // and let each of the newNodes's styles control whether to be highlighted or transparent. + // If ancestor was highlighted but one of its new descendant nodes was transparent, + // all that would happen is the descendant would allow the ancestor's highlight color to show through, + // which doesn't achieve what we want :( + startingSpan.classList.add(kDisableHighlightClass); } +} - private makeHighlightedSpan(textContent: string) { - const newSpan = document.createElement("span"); - newSpan.classList.add(kEnableHighlightClass); - newSpan.appendChild(document.createTextNode(textContent)); - return newSpan; - } +function makeHighlightedSpan(textContent: string) { + const newSpan = document.createElement("span"); + newSpan.classList.add(kEnableHighlightClass); + newSpan.appendChild(document.createTextNode(textContent)); + return newSpan; +} - private playCurrentInternal() { - if (BloomPlayerCore.currentPlaybackMode === PlaybackMode.AudioPlaying) { - const mediaPlayer = this.getPlayer(); - if (mediaPlayer) { - const element = this.playerPage.querySelector( - `#${this.currentAudioId}` - ); - if (!element || !this.canPlayAudio(element)) { - this.playEnded(); - return; - } +function playCurrentInternal() { + if (currentPlaybackMode === PlaybackMode.AudioPlaying) { + const mediaPlayer = getPlayer(); + if (mediaPlayer) { + const element = getCurrentPage().querySelector( + `#${currentAudioId}` + ); + if (!element || !canPlayAudio(element)) { + playEnded(); + return; + } - // Regardless of whether we end up using timingsStr or not, - // we should reset this now in case the previous page used it and was still playing - // when the user flipped to the next page. - this.subElementsWithTimings = []; + // Regardless of whether we end up using timingsStr or not, + // we should reset this now in case the previous page used it and was still playing + // when the user flipped to the next page. + subElementsWithTimings = []; - const timingsStr: string | null = element.getAttribute( - "data-audioRecordingEndTimes" + const timingsStr: string | null = element.getAttribute( + "data-audioRecordingEndTimes" + ); + if (timingsStr) { + const childSpanElements = element.querySelectorAll( + `span.${kSegmentClass}` + ); + const fields = timingsStr.split(" "); + const subElementCount = Math.min( + fields.length, + childSpanElements.length ); - if (timingsStr) { - const childSpanElements = element.querySelectorAll( - `span.${kSegmentClass}` - ); - const fields = timingsStr.split(" "); - const subElementCount = Math.min( - fields.length, - childSpanElements.length - ); - - for (let i = subElementCount - 1; i >= 0; --i) { - const durationSecs: number = Number(fields[i]); - if (isNaN(durationSecs)) { - continue; - } - this.subElementsWithTimings.push([ - childSpanElements.item(i), - durationSecs - ]); - } - } else { - // No timings string available. - // No need for us to do anything. The correct element is already highlighted by playAllSentences() (which needed to call setCurrent... anyway to set the audio player source). - // We'll just proceed along, start playing the audio, and playNextSubElement() will return immediately because there are no sub-elements in this case. - } - const currentSegment = element as HTMLElement; - if (currentSegment && this.ToggleImageDescription) { - this.ToggleImageDescription.raise( - this.isImageDescriptionSegment(currentSegment) - ); + for (let i = subElementCount - 1; i >= 0; --i) { + const durationSecs: number = Number(fields[i]); + if (isNaN(durationSecs)) { + continue; + } + subElementsWithTimings.push([ + childSpanElements.item(i), + durationSecs + ]); } + } else { + // No timings string available. + // No need for us to do anything. The correct element is already highlighted by playAllSentences() (which needed to call setCurrent... anyway to set the audio player source). + // We'll just proceed along, start playing the audio, and playNextSubElement() will return immediately because there are no sub-elements in this case. + } - const promise = mediaPlayer.play(); - ++this.currentAudioSessionNum; - this.audioPlayCurrentStartTime = new Date().getTime(); - this.highlightNextSubElement(this.currentAudioSessionNum); - this.handlePlayPromise(promise); + const currentSegment = element as HTMLElement; + if (currentSegment && ToggleImageDescription) { + ToggleImageDescription?.raise( + isImageDescriptionSegment(currentSegment) + ); } + + gotErrorPlaying = false; + const promise = mediaPlayer.play(); + ++currentAudioSessionNum; + audioPlayCurrentStartTime = new Date().getTime(); + highlightNextSubElement(currentAudioSessionNum); + handlePlayPromise(promise); } } +} - private isImageDescriptionSegment(segment: HTMLElement): boolean { - return segment.closest("." + kImageDescriptionClass) !== null; - } +function isImageDescriptionSegment(segment: HTMLElement): boolean { + return segment.closest("." + kImageDescriptionClass) !== null; +} - private handlePlayPromise(promise: Promise) { - // In newer browsers, play() returns a promise which fails - // if the browser disobeys the command to play, as some do - // if the user hasn't 'interacted' with the page in some - // way that makes the browser think they are OK with it - // playing audio. In Gecko45, the return value is undefined, - // so we mustn't call catch. - if (promise && promise.catch) { - promise.catch((reason: any) => { - // There is an error handler here, but the HTMLMediaElement also has an error handler (which will end up calling playEnded()). - // This promise.catch error handler is the only one that handles NotAllowedException (that is, playback not started because user has not interacted with the page yet). - // However, older versions of browsers don't support promise from HTMLMediaElement.play(). So this cannot be the only error handler. - // Thus we need both the promise.catch error handler as well as the HTMLMediaElement's error handler. - // - // In many cases (such as NotSupportedError, which happens when the audio file isn't found), both error handlers will run. - // That is a little annoying but if the two don't conflict with each other it's not problematic. - - console.log("could not play sound: " + reason); - - if ( - reason && - reason - .toString() - .includes( - "The play() request was interrupted by a call to pause()." - ) - ) { - // We were getting this error Aug 2020. I tried wrapping the line above which calls mediaPlayer.play() - // (currently `promise = mediaPlayer.play();`) in a setTimeout with 0ms. This seemed to fix the bug (with - // landscape books not having audio play initially -- BL-8887). But the root cause was actually that - // we ended up calling playAllSentences twice when the book first loaded. - // I fixed that in bloom-player-core. But I wanted to document the possible setTimeout fix here - // in case this issue ever comes up for a different reason. - console.log( - "See comment in narration.ts for possibly useful information regarding this error." - ); - } +function handlePlayPromise(promise: Promise, player?: HTMLMediaElement) { + // In newer browsers, play() returns a promise which fails + // if the browser disobeys the command to play, as some do + // if the user hasn't 'interacted' with the page in some + // way that makes the browser think they are OK with it + // playing audio. In Gecko45, the return value is undefined, + // so we mustn't call catch. + if (promise && promise.catch) { + promise.catch((reason: any) => { + // The HTMLMediaElement also has an error handler (which calls playEnded()). + // We do NOT want to call that here, both to stop it happening twice, but also because + // we do NOT want to call playEnded (which in autoplay causes advance to next page) + // when we get NotAllowedError. That error seems to only come here, and not to raise the + // ended event. + + // This promise.catch error handler is the only one that handles NotAllowedException (that is, playback not started because user has not interacted with the page yet). + // However, older versions of browsers don't support promise from HTMLMediaElement.play(). So this cannot be the only error handler. + // Thus we need both the promise.catch error handler as well as the HTMLMediaElement's error handler. + // + // In many cases (such as NotSupportedError, which happens when the audio file isn't found), both error handlers will run. + // That is a little annoying but if the two don't conflict with each other it's not problematic. + + const playingWhat = player?.getAttribute("src") ?? "unknown"; + console.log("could not play sound: " + reason + " " + playingWhat); + + if ( + reason && + reason + .toString() + .includes( + "The play() request was interrupted by a call to pause()." + ) + ) { + // We were getting this error Aug 2020. I tried wrapping the line above which calls mediaPlayer.play() + // (currently `promise = mediaPlayer.play();`) in a setTimeout with 0ms. This seemed to fix the bug (with + // landscape books not having audio play initially -- BL-8887). But the root cause was actually that + // we ended up calling playAllSentences twice when the book first loaded. + // I fixed that in bloom-player-core. But I wanted to document the possible setTimeout fix here + // in case this issue ever comes up for a different reason. + console.log( + "See comment in narration.ts for possibly useful information regarding this error." + ); + } - // Don't call removeAudioCurrent() here. The HTMLMediaElement's error handler will call playEnded() and calling removeAudioCurrent() here will mess up playEnded(). - // this.removeAudioCurrent(); - - // With some kinds of invalid sound file it keeps trying and plays over and over. - this.getPlayer().pause(); - // if (this.Pause) { - // this.Pause.raise(); - // } - - // Get all the state (and UI) set correctly again. - // Not entirely sure about limiting this to NotAllowedError, but that's - // the one kind of play error that is fixed by the user just interacting. - // If there's some other reason we can't play, showing as paused may not - // be useful. See comments on the similar code in music.ts - if (reason.name === "NotAllowedError" && this.PlayFailed) { - this.PlayFailed.raise(); - } - }); - } + // Don't call removeAudioCurrent() here. The HTMLMediaElement's error handler will call playEnded() and calling removeAudioCurrent() here will mess up playEnded(). + // removeAudioCurrent(); + + // With some kinds of invalid sound file it keeps trying and plays over and over. + // But when we move on to play another sound, a pause here will mess things up. + // So instead I put a pause after we run out of sounds to try to play. + //getPlayer().pause(); + // if (Pause) { + // Pause?.raise(); + // } + + // Get all the state (and UI) set correctly again. + // Not entirely sure about limiting this to NotAllowedError, but that's + // the one kind of play error that is fixed by the user just interacting. + // If there's some other reason we can't play, showing as paused may not + // be useful. See comments on the similar code in music.ts + if (reason.name === "NotAllowedError" && PlayFailed) { + PlayFailed?.raise(); + } + }); } +} - // Moves the highlight to the next sub-element - // originalSessionNum: The value of this.currentAudioSessionNum at the time when the audio file started playing. - // This is used to check in the future if the timeouts we started are for the right session - // startTimeInSecs is an optional fallback that will be used in case the currentTime cannot be determined from the audio player element. - private highlightNextSubElement( - originalSessionNum: number, - startTimeInSecs: number = 0 - ) { - // the item should not be popped off the stack until it's completely done with. - const subElementCount = this.subElementsWithTimings.length; - - if (subElementCount <= 0) { - return; - } - - const topTuple = this.subElementsWithTimings[subElementCount - 1]; - const element = topTuple[0]; - const endTimeInSecs: number = topTuple[1]; +// Moves the highlight to the next sub-element +// originalSessionNum: The value of currentAudioSessionNum at the time when the audio file started playing. +// This is used to check in the future if the timeouts we started are for the right session. +// startTimeInSecs is an optional fallback that will be used in case the currentTime cannot be determined from the audio player element. +function highlightNextSubElement( + originalSessionNum: number, + startTimeInSecs: number = 0 +) { + // the item should not be popped off the stack until it's completely done with. + const subElementCount = subElementsWithTimings.length; + + if (subElementCount <= 0) { + return; + } - this.setHighlightTo({ - newElement: element, - shouldScrollToElement: true, - disableHighlightIfNoAudio: false - }); + const topTuple = subElementsWithTimings[subElementCount - 1]; + const element = topTuple[0]; + const endTimeInSecs: number = topTuple[1]; + + setHighlightTo({ + newElement: element, + shouldScrollToElement: true, + disableHighlightIfNoAudio: false + }); + + const mediaPlayer: HTMLMediaElement = document.getElementById( + "bloom-audio-player" + )! as HTMLMediaElement; + let currentTimeInSecs: number = mediaPlayer.currentTime; + if (currentTimeInSecs <= 0) { + currentTimeInSecs = startTimeInSecs; + } - const mediaPlayer: HTMLMediaElement = document.getElementById( - "bloom-audio-player" - )! as HTMLMediaElement; - let currentTimeInSecs: number = mediaPlayer.currentTime; - if (currentTimeInSecs <= 0) { - currentTimeInSecs = startTimeInSecs; - } + // Handle cases where the currentTime has already exceeded the nextStartTime + // (might happen if you're unlucky in the thread queue... or if in debugger, etc.) + // But instead of setting time to 0, set the minimum highlight time threshold to 0.1 (this threshold is arbitrary). + const durationInSecs = Math.max(endTimeInSecs - currentTimeInSecs, 0.1); - // Handle cases where the currentTime has already exceeded the nextStartTime - // (might happen if you're unlucky in the thread queue... or if in debugger, etc.) - // But instead of setting time to 0, set the minimum highlight time threshold to 0.1 (this threshold is arbitrary). - const durationInSecs = Math.max(endTimeInSecs - currentTimeInSecs, 0.1); + setTimeout(() => { + onSubElementHighlightTimeEnded(originalSessionNum); + }, durationInSecs * 1000); +} - setTimeout(() => { - this.onSubElementHighlightTimeEnded(originalSessionNum); - }, durationInSecs * 1000); +// Handles a timeout indicating that the expected time for highlighting the current subElement has ended. +// If we've really played to the end of that subElement, highlight the next one (if any). +// originalSessionNum: The value of currentAudioSessionNum at the time when the audio file started playing. +// This is used to check in the future if the timeouts we started are for the right session +function onSubElementHighlightTimeEnded(originalSessionNum: number) { + // Check if the user has changed pages since the original audio for this started playing. + // Note: Using the timestamp allows us to detect switching to the next page and then back to this page. + // Using playerPage (HTMLElement) does not detect that. + if (originalSessionNum !== currentAudioSessionNum) { + return; + } + // Seems to be needed to prevent jumping to the next subelement when not permitted to play by browser. + // Not sure why the check below on mediaPlayer.currentTime does not prevent this. + if (currentPlaybackMode === PlaybackMode.AudioPaused) { + return; } - // Handles a timeout indicating that the expected time for highlighting the current subElement has ended. - // If we've really played to the end of that subElement, highlight the next one (if any). - // originalSessionNum: The value of this.currentAudioSessionNum at the time when the audio file started playing. - // This is used to check in the future if the timeouts we started are for the right session - private onSubElementHighlightTimeEnded(originalSessionNum: number) { - // Check if the user has changed pages since the original audio for this started playing. - // Note: Using the timestamp allows us to detect switching to the next page and then back to this page. - // Using this.playerPage (HTMLElement) does not detect that. - if (originalSessionNum !== this.currentAudioSessionNum) { - return; - } - // Seems to be needed to prevent jumping to the next subelement when not permitted to play by browser. - // Not sure why the check below on mediaPlayer.currentTime does not prevent this. - if (BloomPlayerCore.currentPlaybackMode === PlaybackMode.AudioPaused) { - return; - } - - const subElementCount = this.subElementsWithTimings.length; - if (subElementCount <= 0) { - return; - } + const subElementCount = subElementsWithTimings.length; + if (subElementCount <= 0) { + return; + } - const mediaPlayer: HTMLMediaElement = document.getElementById( - "bloom-audio-player" - )! as HTMLMediaElement; - if (mediaPlayer.ended || mediaPlayer.error) { - // audio playback ended. No need to highlight anything else. - // (No real need to remove the highlights either, because playEnded() is supposed to take care of that.) - return; - } - const playedDurationInSecs: number | undefined | null = - mediaPlayer.currentTime; - - // Peek at the next sentence and see if we're ready to start that one. (We might not be ready to play the next audio if the current audio got paused). - const subElementWithTiming = this.subElementsWithTimings[ - subElementCount - 1 - ]; - const nextStartTimeInSecs = subElementWithTiming[1]; - - if ( - playedDurationInSecs && - playedDurationInSecs < nextStartTimeInSecs - ) { - // Still need to wait. Exit this function early and re-check later. - const minRemainingDurationInSecs = - nextStartTimeInSecs - playedDurationInSecs; - setTimeout(() => { - this.onSubElementHighlightTimeEnded(originalSessionNum); - }, minRemainingDurationInSecs * 1000); + const mediaPlayer: HTMLMediaElement = document.getElementById( + "bloom-audio-player" + )! as HTMLMediaElement; + if (mediaPlayer.ended || mediaPlayer.error) { + // audio playback ended. No need to highlight anything else. + // (No real need to remove the highlights either, because playEnded() is supposed to take care of that.) + return; + } + const playedDurationInSecs: number | undefined | null = + mediaPlayer.currentTime; - return; - } + // Peek at the next sentence and see if we're ready to start that one. (We might not be ready to play the next audio if the current audio got paused). + const subElementWithTiming = subElementsWithTimings[subElementCount - 1]; + const nextStartTimeInSecs = subElementWithTiming[1]; - this.subElementsWithTimings.pop(); + if (playedDurationInSecs && playedDurationInSecs < nextStartTimeInSecs) { + // Still need to wait. Exit this function early and re-check later. + const minRemainingDurationInSecs = + nextStartTimeInSecs - playedDurationInSecs; + setTimeout(() => { + onSubElementHighlightTimeEnded(originalSessionNum); + }, minRemainingDurationInSecs * 1000); - this.highlightNextSubElement(originalSessionNum, nextStartTimeInSecs); + return; } - // Removes the .ui-audioCurrent class from all elements (also ui-audioCurrentImg) - // Equivalent of removeAudioCurrentFromPageDocBody() in BloomDesktop. - private removeAudioCurrent() { - // Note that HTMLCollectionOf's length can change if you change the number of elements matching the selector. - const audioCurrentCollection: HTMLCollectionOf = document.getElementsByClassName( - "ui-audioCurrent" - ); + subElementsWithTimings.pop(); - // Convert to an array whose length won't be changed - const audioCurrentArray: Element[] = Array.from(audioCurrentCollection); + highlightNextSubElement(originalSessionNum, nextStartTimeInSecs); +} - for (let i = 0; i < audioCurrentArray.length; i++) { - audioCurrentArray[i].classList.remove("ui-audioCurrent"); - } - const currentImg = document.getElementsByClassName( - "ui-audioCurrentImg" - )[0]; - if (currentImg) { - currentImg.classList.remove("ui-audioCurrentImg"); - } +// Removes the .ui-audioCurrent class from all elements (also ui-audioCurrentImg) +// Equivalent of removeAudioCurrentFromPageDocBody() in BloomDesktop. +function removeAudioCurrent(around: HTMLElement = document.body) { + // Note that HTMLCollectionOf's length can change if you change the number of elements matching the selector. + // For safety we get rid of all existing ones. But we do take a starting point element + // (might be the one that has the higlight, or the one getting it) + // to make sure we're cleaning up in the right document, which is in question when used in + // Bloom Editor. + const audioCurrentArray = Array.from( + around.ownerDocument.getElementsByClassName("ui-audioCurrent") + ); + + for (let i = 0; i < audioCurrentArray.length; i++) { + audioCurrentArray[i].classList.remove("ui-audioCurrent"); } - - private setSoundAndHighlight( - newElement: Element, - disableHighlightIfNoAudio: boolean, - oldElement?: Element | null | undefined - ) { - this.setHighlightTo({ - newElement, - shouldScrollToElement: true, // Always true in bloom-player version - disableHighlightIfNoAudio, - oldElement - }); - this.setSoundFrom(newElement); + const currentImg = document.getElementsByClassName("ui-audioCurrentImg")[0]; + if (currentImg) { + currentImg.classList.remove("ui-audioCurrentImg"); } +} - private setHighlightTo({ +function setSoundAndHighlight( + newElement: Element, + disableHighlightIfNoAudio: boolean, + oldElement?: Element | null | undefined +) { + setHighlightTo({ newElement, - shouldScrollToElement, + shouldScrollToElement: true, // Always true in bloom-player version disableHighlightIfNoAudio, oldElement - }: ISetHighlightParams) { - // This should happen even if oldElement and newElement are the same. - if (shouldScrollToElement) { - // Wrap it in a try/catch so that if something breaks with this minor/nice-to-have feature of scrolling, - // the main responsibilities of this method can still proceed - try { - this.scrollElementIntoView(newElement); - } catch (e) { - console.error(e); - } - } - - if (oldElement === newElement) { - // No need to do much, and better not to, so that we can avoid any temporary flashes as the highlight is removed and re-applied - return; - } + }); + setSoundFrom(newElement); +} - this.removeAudioCurrent(); - - if (disableHighlightIfNoAudio) { - const mediaPlayer = this.getPlayer(); - const isAlreadyPlaying = mediaPlayer.currentTime > 0; - - // If it's already playing, no need to disable (Especially in the Soft Split case, where only one file is playing but multiple sentences need to be highlighted). - if (!isAlreadyPlaying) { - // Start off in a highlight-disabled state so we don't display any momentary highlight for cases where there is no audio for this element. - // In react-based bloom-player, canPlayAudio() can't trivially identify whether or not audio exists, - // so we need to incorporate a derivative of Bloom Desktop's .ui-suppressHighlight code - newElement.classList.add(kSuppressHighlightClass); - mediaPlayer.addEventListener("playing", () => { - newElement.classList.remove(kSuppressHighlightClass); - }); - } +function setHighlightTo({ + newElement, + shouldScrollToElement, + disableHighlightIfNoAudio, + oldElement +}: ISetHighlightParams) { + // This should happen even if oldElement and newElement are the same. + if (shouldScrollToElement) { + // Wrap it in a try/catch so that if something breaks with this minor/nice-to-have feature of scrolling, + // the main responsibilities of this method can still proceed + try { + scrollElementIntoView(newElement); + } catch (e) { + console.error(e); } + } - newElement.classList.add("ui-audioCurrent"); - // If the current audio is part of a (currently typically hidden) image description, - // highlight the image. - // it's important to check for imageDescription on the translationGroup; - // we don't want to highlight the image while, for example, playing a TOP box content. - const translationGroup = newElement.closest(".bloom-translationGroup"); - if ( - translationGroup && - translationGroup.classList.contains(kImageDescriptionClass) - ) { - const imgContainer = translationGroup.closest( - ".bloom-imageContainer" - ); - if (imgContainer) { - imgContainer.classList.add("ui-audioCurrentImg"); - } - } + if (oldElement === newElement) { + // No need to do much, and better not to, so that we can avoid any temporary flashes as the highlight is removed and re-applied + return; } - // Scrolls an element into view. - private scrollElementIntoView(element: Element) { - // In Bloom Player, scrollIntoView can interfere with page swipes, - // so Bloom Player needs some smarts about when to call it... - if (this.isSwipeInProgress()) { - // This alternative implementation doesn't use scrollIntoView (Which interferes with swiper). - // Since swiping is only active at the beginning (usually while the 1st element is playing) - // it should generally be good enough just to reset the scroll of the scroll parent to the top. - - // Assumption: Assumes the editable is the scrollbox. - // If this is not the case, you can use JQuery's scrollParent() function or other equivalent - const scrollAncestor = this.getEditable(element); - if (scrollAncestor) { - scrollAncestor.scrollTop = 0; - } - return; + removeAudioCurrent((oldElement || newElement) as HTMLElement); + + if (disableHighlightIfNoAudio) { + const mediaPlayer = getPlayer(); + const isAlreadyPlaying = mediaPlayer.currentTime > 0; + + // If it's already playing, no need to disable (Especially in the Soft Split case, where only one file is playing but multiple sentences need to be highlighted). + if (!isAlreadyPlaying) { + // Start off in a highlight-disabled state so we don't display any momentary highlight for cases where there is no audio for this element. + // In react-based bloom-player, canPlayAudio() can't trivially identify whether or not audio exists, + // so we need to incorporate a derivative of Bloom Desktop's .ui-suppressHighlight code + newElement.classList.add(kSuppressHighlightClass); + // When it starts playing, we know we really have such an audio file, so we can stop + // suppressing the highlight. + mediaPlayer.addEventListener("playing", () => { + newElement.classList.remove(kSuppressHighlightClass); + }); } + } - let mover = element as HTMLElement; // by default make the element itself scrollIntoView - if ( - window.getComputedStyle(element.parentElement!).position !== - "static" - ) { - // We can make a new element absolutely positioned and it will be relative to the parent. - // The idea is to make an element much narrower than the element we are - // trying to make visible, since we don't want horizontal movement. Quite possibly, - // as in BL-11038, only some white space is actually off-screen. But even if the author - // has positioned a bubble so some text is cut off, we don't want horizontal scrolling, - // which inside swiper will weirdly pull in part of the next page. - // (In the pathological case that the bubble is more than half hidden, we'll do the - // horizontal scroll, despite the ugliness of possibly showing part of the next page.) - // Note that elt may be a span, when scrolling chunks of text into view to play. - // I thought about using scrollWidth/Height to include any part of the element - // that is scrolled out of view, but for some reason these are always zero for spans. - // OffsetHeight seems to give the full height, though docs seem to indicate that it - // should not include invisible areas. - const elt = element as HTMLElement; - mover = document.createElement("div"); - mover.style.position = "absolute"; - mover.style.top = elt.offsetTop + "px"; - - // now we need what for a block would be offsetLeft. However, for a span, that - // yields the offset of the top left corner, which may be in the middle - // of a line. - const bounds = elt.getBoundingClientRect(); - const parent = elt.parentElement; - const parentBounds = parent?.getBoundingClientRect(); - const scale = parentBounds!.width / parent!.offsetWidth; - const leftRelativeToParent = - (bounds.left - parentBounds!.left) / scale; - - mover.style.left = - leftRelativeToParent + elt.offsetWidth / 2 + "px"; - mover.style.height = elt.offsetHeight + "px"; - mover.style.width = "0"; - element.parentElement?.insertBefore(mover, element); + newElement.classList.add("ui-audioCurrent"); + // If the current audio is part of a (currently typically hidden) image description, + // highlight the image. + // it's important to check for imageDescription on the translationGroup; + // we don't want to highlight the image while, for example, playing a TOP box content. + const translationGroup = newElement.closest(".bloom-translationGroup"); + if ( + translationGroup && + translationGroup.classList.contains(kImageDescriptionClass) + ) { + const imgContainer = translationGroup.closest(".bloom-imageContainer"); + if (imgContainer) { + imgContainer.classList.add("ui-audioCurrentImg"); } + } +} - mover.scrollIntoView({ - // Animated instead of sudden - behavior: "smooth", - - // "nearest" setting does lots of smarts for us (compared to us deciding when to use "start" or "end") - // Seems to reduce unnecessary scrolling compared to start (aka true) or end (aka false). - // Refer to https://drafts.csswg.org/cssom-view/#scroll-an-element-into-view, - // which seems to imply that it won't do any scrolling if the two relevant edges are already inside. - block: "nearest" - - // horizontal alignment is controlled by "inline". We'll leave it as its default ("nearest") - // which typically won't move things at all horizontally - }); - if (mover !== element) { - mover.parentElement?.removeChild(mover); +// Scrolls an element into view. +function scrollElementIntoView(element: Element) { + // In Bloom Player, scrollIntoView can interfere with page swipes, + // so Bloom Player needs some smarts about when to call it... + if (isSwipeInProgress?.() || recentPageChange) { + // This alternative implementation doesn't use scrollIntoView (Which interferes with swiper). + // Since swiping is only active at the beginning (usually while the 1st element is playing) + // it should generally be good enough just to reset the scroll of the scroll parent to the top. + + // Assumption: Assumes the editable is the scrollbox. + // If this is not the case, you can use JQuery's scrollParent() function or other equivalent + const scrollAncestor = getEditable(element); + if (scrollAncestor) { + scrollAncestor.scrollTop = 0; } + return; } - // Returns true if swiping to this page is still in progress. - private isSwipeInProgress(): boolean { - return this.swiperInstance && this.swiperInstance.animating; + let mover = element as HTMLElement; // by default make the element itself scrollIntoView + if (window.getComputedStyle(element.parentElement!).position !== "static") { + // We can make a new element absolutely positioned and it will be relative to the parent. + // The idea is to make an element much narrower than the element we are + // trying to make visible, since we don't want horizontal movement. Quite possibly, + // as in BL-11038, only some white space is actually off-screen. But even if the author + // has positioned a bubble so some text is cut off, we don't want horizontal scrolling, + // which inside swiper will weirdly pull in part of the next page. + // (In the pathological case that the bubble is more than half hidden, we'll do the + // horizontal scroll, despite the ugliness of possibly showing part of the next page.) + // Note that elt may be a span, when scrolling chunks of text into view to play. + // I thought about using scrollWidth/Height to include any part of the element + // that is scrolled out of view, but for some reason these are always zero for spans. + // OffsetHeight seems to give the full height, though docs seem to indicate that it + // should not include invisible areas. + const elt = element as HTMLElement; + mover = document.createElement("div"); + mover.style.position = "absolute"; + mover.style.top = elt.offsetTop + "px"; + + // now we need what for a block would be offsetLeft. However, for a span, that + // yields the offset of the top left corner, which may be in the middle + // of a line. + const bounds = elt.getBoundingClientRect(); + const parent = elt.parentElement; + const parentBounds = parent?.getBoundingClientRect(); + const scale = parentBounds!.width / parent!.offsetWidth; + const leftRelativeToParent = (bounds.left - parentBounds!.left) / scale; + + mover.style.left = leftRelativeToParent + elt.offsetWidth / 2 + "px"; + mover.style.height = elt.offsetHeight + "px"; + mover.style.width = "0"; + element.parentElement?.insertBefore(mover, element); } - private getEditable(element: Element): Element | null { - if (element.classList.contains("bloom-editable")) { - return element; - } else { - return element.closest(".bloom-editable"); // Might be null - } + mover.scrollIntoView({ + // Animated instead of sudden + behavior: "smooth", + + // "nearest" setting does lots of smarts for us (compared to us deciding when to use "start" or "end") + // Seems to reduce unnecessary scrolling compared to start (aka true) or end (aka false). + // Refer to https://drafts.csswg.org/cssom-view/#scroll-an-element-into-view, + // which seems to imply that it won't do any scrolling if the two relevant edges are already inside. + block: "nearest" + + // horizontal alignment is controlled by "inline". We'll leave it as its default ("nearest") + // which typically won't move things at all horizontally + }); + if (mover !== element) { + mover.parentElement?.removeChild(mover); } +} - private setSoundFrom(element: Element) { - const firstAudioSentence = this.getFirstAudioSentenceWithinElement( - element - ); - const id: string = firstAudioSentence - ? firstAudioSentence.id - : element.id; - this.setCurrentAudioId(id); +function getEditable(element: Element): Element | null { + if (element.classList.contains("bloom-editable")) { + return element; + } else { + return element.closest(".bloom-editable"); // Might be null } +} - public getFirstAudioSentenceWithinElement( - element: Element | null - ): Element | null { - const audioSentences = this.getAudioSegmentsWithinElement(element); - if (!audioSentences || audioSentences.length === 0) { - return null; - } +function setSoundFrom(element: Element) { + const firstAudioSentence = getFirstAudioSentenceWithinElement(element); + const id: string = firstAudioSentence ? firstAudioSentence.id : element.id; + setCurrentAudioId(id); +} - return audioSentences[0]; +function getFirstAudioSentenceWithinElement( + element: Element | null +): Element | null { + const audioSentences = getAudioSegmentsWithinElement(element); + if (!audioSentences || audioSentences.length === 0) { + return null; } - public getAudioSegmentsWithinElement(element: Element | null): Element[] { - const audioSegments: Element[] = []; + return audioSentences[0]; +} - if (element) { - if (element.classList.contains(kAudioSentence)) { - audioSegments.push(element); - } else { - const collection = element.getElementsByClassName( - kAudioSentence - ); - for (let i = 0; i < collection.length; ++i) { - const audioSentenceElement = collection.item(i); - if (audioSentenceElement) { - audioSegments.push(audioSentenceElement); - } +function getAudioSegmentsWithinElement(element: Element | null): Element[] { + const audioSegments: Element[] = []; + + if (element) { + if (element.classList.contains(kAudioSentence)) { + audioSegments.push(element); + } else { + const collection = element.getElementsByClassName(kAudioSentence); + for (let i = 0; i < collection.length; ++i) { + const audioSentenceElement = collection.item(i); + if (audioSentenceElement) { + audioSegments.push(audioSentenceElement); } } } - - return audioSegments; } - // Setter for currentAudio - public setCurrentAudioId(id: string) { - if (!this.currentAudioId || this.currentAudioId !== id) { - this.currentAudioId = id; - this.updatePlayerStatus(); - } - } + return audioSegments; +} - private updatePlayerStatus() { - const player = this.getPlayer(); - if (!player) { - return; - } - // Any time we change the src, the player will pause. - // So if we're playing currently, we'd better report whatever time - // we played. - if (player.currentTime > 0 && !player.paused && !player.ended) { - this.reportPlayDuration(); - } - const url = this.currentAudioUrl(this.currentAudioId); - logSound(url, 1); - player.setAttribute("src", url + "?nocache=" + new Date().getTime()); +function setCurrentAudioId(id: string) { + if (!currentAudioId || currentAudioId !== id) { + currentAudioId = id; + updatePlayerStatus(); } +} - private currentAudioUrl(id: string): string { - return this.urlPrefix + "/audio/" + id + ".mp3"; +function updatePlayerStatus() { + const player = getPlayer(); + if (!player) { + return; } - - private playEndedFunction = () => { - this.playEnded(); - }; - - private getPlayer(): HTMLMediaElement { - const audio = this.getAudio("bloom-audio-player", audio => {}); - // We used to do this in the init call, but sometimes the function didn't get called. - // Suspecting that there are cases, maybe just in storybook, where a new instance - // of the narration object gets created, but the old audio element still exists. - // Make sure the current instance has our end function. - // Because it is a fixed function for the lifetime of this object, addEventListener - // will not add it repeatedly. - audio.addEventListener("ended", this.playEndedFunction); - audio.addEventListener("error", this.playEndedFunction); - return audio; + // Any time we change the src, the player will pause. + // So if we're playing currently, we'd better report whatever time + // we played. + if (player.currentTime > 0 && !player.paused && !player.ended) { + reportPlayDuration(); } + const url = currentAudioUrl(currentAudioId); + logNarration(url); + // because this code is meant to work in both Bloom and BloomPlayer, we can't call a Bloom API to find + // out whether we actually have a recording (as we well might not, if we just opened the talking book + // tool and haven't recorded anything yet). So we just try to play it and see what happens. + // The optional param tells Bloom not to report an error if the file isn't found, and is ignored in + // other contexts. + player.setAttribute( + "src", + url + "?nocache=" + new Date().getTime() + "&optional=true" + ); +} - public playEnded(): void { - // Not sure if this is necessary, since both 'playCurrentInternal()' and 'reportPlayEnded()' - // will toggle image description already, but if we've just gotten to the end of our "stack", - // it may be needed. - if (this.ToggleImageDescription) { - this.ToggleImageDescription.raise(false); - } - this.reportPlayDuration(); - if ( - this.elementsToPlayConsecutivelyStack && - this.elementsToPlayConsecutivelyStack.length > 0 - ) { - this.elementsToPlayConsecutivelyStack.pop(); // get rid of the last one we played - const newStackCount = this.elementsToPlayConsecutivelyStack.length; - if (newStackCount > 0) { - // More items to play - const nextElement = this.elementsToPlayConsecutivelyStack[ - newStackCount - 1 - ]; - this.setSoundAndHighlight(nextElement, true); - this.playCurrentInternal(); - } else { - // Nothing left to play - this.reportPlayEnded(); - } - } - } +function currentAudioUrl(id: string): string { + const result = urlPrefix() + "/audio/" + id + ".mp3"; + return result; +} - private reportPlayEnded() { - this.elementsToPlayConsecutivelyStack = []; - this.subElementsWithTimings = []; +function getPlayer(): HTMLMediaElement { + const audio = getAudio("bloom-audio-player", a => { + a.addEventListener("ended", playEnded); + a.addEventListener("error", handlePlayError); + }); + return audio; +} - this.removeAudioCurrent(); - if (this.PageNarrationComplete) { - this.PageNarrationComplete.raise(this.playerPage); - } - if (this.PlayCompleted) { - this.PlayCompleted.raise(); - } +function playEnded(): void { + // Not sure if this is necessary, since both 'playCurrentInternal()' and 'reportPlayEnded()' + // will toggle image description already, but if we've just gotten to the end of our "stack", + // it may be needed. + if (ToggleImageDescription) { + ToggleImageDescription?.raise(false); } - - private reportPlayDuration() { - if (!this.audioPlayCurrentStartTime) { - return; + reportPlayDuration(); + if ( + elementsToPlayConsecutivelyStack && + elementsToPlayConsecutivelyStack.length > 0 + ) { + const elementJustPlayed = elementsToPlayConsecutivelyStack.pop(); // get rid of the last one we played + const newStackCount = elementsToPlayConsecutivelyStack.length; + if (newStackCount > 0) { + // More items to play + const nextElement = + elementsToPlayConsecutivelyStack[newStackCount - 1]; + setSoundAndHighlight(nextElement, true); + playCurrentInternal(); + } else { + reportPlayEnded(); + removeAudioCurrent(elementJustPlayed); + // In some error conditions, we need to stop repeating attempts to play. + getPlayer().pause(); } - const currentTime = new Date().getTime(); - const duration = (currentTime - this.audioPlayCurrentStartTime) / 1000; - BloomPlayerCore.storeAudioAnalytics(duration); } +} - private getAudio(id: string, init: (audio: HTMLAudioElement) => void) { - let player: HTMLAudioElement | null = document.querySelector( - "#" + id - ) as HTMLAudioElement; - if (player && !player.play) { - player.remove(); - player = null; - } - if (!player) { - player = document.createElement("audio") as HTMLAudioElement; - player.setAttribute("id", id); - document.body.appendChild(player); - init(player); - } - return player as HTMLMediaElement; - } +function reportPlayEnded() { + elementsToPlayConsecutivelyStack = []; + subElementsWithTimings = []; - public canPlayAudio(current: Element): boolean { - return true; // currently no way to check + removeAudioCurrent(); + PageNarrationComplete?.raise(currentPlayPage!); + PlayCompleted?.raise(); +} + +function reportPlayDuration() { + if (!audioPlayCurrentStartTime || !durationReporter) { + return; } + const currentTime = new Date().getTime(); + const duration = (currentTime - audioPlayCurrentStartTime) / 1000; + durationReporter(duration); +} - public setIncludeImageDescriptions(includeImageDescriptions: boolean) { - this.includeImageDescriptions = includeImageDescriptions; +function getAudio(id: string, init: (audio: HTMLAudioElement) => void) { + let player: HTMLAudioElement | null = document.querySelector( + "#" + id + ) as HTMLAudioElement; + // If (somehow?) it exists but is not a valid HTMLAudioElement, remove it. + if (player && !player.play) { + player.remove(); + player = null; + } + if (!player) { + player = document.createElement("audio") as HTMLAudioElement; + player.setAttribute("id", id); + document.body.appendChild(player); + init(player); } + return player as HTMLMediaElement; +} - // Returns all elements that match CSS selector {expr} as an array. - // Querying can optionally be restricted to {container}’s descendants - // If includeSelf is true, it includes both itself as well as its descendants. - // Otherwise, it only includes descendants. - // Also filters out imageDescriptions if we aren't supposed to be reading them. - private findAll( - expr: string, - container: HTMLElement, - includeSelf: boolean = false - ): HTMLElement[] { - // querySelectorAll checks all the descendants - const allMatches: HTMLElement[] = [].slice.call( - (container || document).querySelectorAll(expr) - ); +function canPlayAudio(current: Element): boolean { + return true; // currently no way to check +} - // Now check itself - if (includeSelf && container && container.matches(expr)) { - allMatches.push(container); - } +// If something goes wrong playing a media element, typically that we don't actually have a recording +// for a particular one, we seem to sometimes get an error event, while other times, the promise returned +// by play() is rejected. Both cases call handlePlayError, which calls playEnded, but in case we get both, +// we don't want to call playEnded twice. +let gotErrorPlaying = false; - return this.includeImageDescriptions - ? allMatches - : allMatches.filter( - match => !this.isImageDescriptionSegment(match) - ); +function handlePlayError() { + if (gotErrorPlaying) { + console.log("Already got error playing, not handling again"); + return; } + gotErrorPlaying = true; + console.log("Error playing, handling"); + setTimeout(() => { + playEnded(); + }, 100); +} - private getPlayableDivs(container: HTMLElement) { - // We want to play any audio we have from divs the user can see. - // This is a crude test, but currently we always use display:none to hide unwanted languages. - return this.findAll(".bloom-editable", container).filter( - e => window.getComputedStyle(e).display !== "none" - ); +// Returns all elements that match CSS selector {expr} as an array. +// Querying can optionally be restricted to {container}'s descendants +// If includeSelf is true, it includes both itself as well as its descendants. +// Otherwise, it only includes descendants. +// Also filters out imageDescriptions if we aren't supposed to be reading them. +function findAll( + expr: string, + container: HTMLElement, + includeSelf: boolean = false +): HTMLElement[] { + // querySelectorAll checks all the descendants + const allMatches: HTMLElement[] = [].slice.call( + (container || document).querySelectorAll(expr) + ); + + // Now check itself + if (includeSelf && container && container.matches(expr)) { + allMatches.push(container); } - // Optional param is for use when 'playerPage' has NOT been initialized. - // Not using the optional param assumes 'playerPage' has been initialized - private getPagePlayableDivs(page?: HTMLElement): HTMLElement[] { - return this.getPlayableDivs(page ? page : this.playerPage); - } + return includeImageDescriptions + ? allMatches + : allMatches.filter(match => !isImageDescriptionSegment(match)); +} - // Optional param is for use when 'playerPage' has NOT been initialized. - // Not using the optional param assumes 'playerPage' has been initialized - private getPageAudioElements(page?: HTMLElement): HTMLElement[] { - return [].concat.apply( - [], - this.getPagePlayableDivs(page).map(x => - this.findAll(".audio-sentence", x, true) - ) - ); - } +function getPlayableDivs(container: HTMLElement) { + // We want to play any audio we have from divs the user can see. + // This is a crude test, but currently we always use display:none to hide unwanted languages. + return findAll(".bloom-editable", container).filter( + e => window.getComputedStyle(e).display !== "none" + ); +} - public pageHasAudio(page: HTMLElement): boolean { - return this.getPageAudioElements(page).length ? true : false; - } +// Optional param is for use when 'playerPage' has NOT been initialized. +// Not using the optional param assumes 'playerPage' has been initialized +function getPagePlayableDivs(page?: HTMLElement): HTMLElement[] { + return getPlayableDivs(page ? page : currentPlayPage!); +} - public play() { - if (BloomPlayerCore.currentPlaybackMode === PlaybackMode.AudioPlaying) { - return; // no change. - } - BloomPlayerCore.currentPlaybackMode = PlaybackMode.AudioPlaying; - // I'm not sure how getPlayer() can return null/undefined, but have seen it happen - // typically when doing something odd like trying to go back from the first page. - if (this.segments.length && this.getPlayer()) { - if (this.elementsToPlayConsecutivelyStack.length) { - this.handlePlayPromise(this.getPlayer().play()); - - // Resuming play. Only currentStartTime needs to be adjusted, but originalStartTime shouldn't be changed. - this.audioPlayCurrentStartTime = new Date().getTime(); - } else { - // Pressing the play button in this case is triggering a replay of the current page, - // so we need to reset the highlighting. - this.playAllSentences(null); - return; - } - } - // adjust startPlay by the elapsed pause. This will cause fakePageNarrationTimedOut to - // start a new timeout if we are depending on it to fake PageNarrationComplete. - const pause = new Date().getTime() - this.startPause.getTime(); - this.startPlay = new Date(this.startPlay.getTime() + pause); - //console.log("paused for " + pause + " and adjusted start time to " + this.startPlay); - if (this.fakeNarrationAborted) { - // we already paused through the timeout for normal advance. - // This call (now we are not paused and have adjusted startPlay) - // will typically start a new timeout. If we are very close to - // the desired duration it may just raise the event at once. - // Either way we should get the event raised exactly once - // at very close to the right time, allowing for pauses. - this.fakeNarrationAborted = false; - this.fakePageNarrationTimedOut(this.playerPage); - } - // in case we're resuming play, we need a new timout when the current subelement is finished - this.highlightNextSubElement(this.currentAudioSessionNum); - } +// Optional param is for use when 'playerPage' has NOT been initialized. +// Not using the optional param assumes 'playerPage' has been initialized +function getPageAudioElements(page?: HTMLElement): HTMLElement[] { + return [].concat.apply( + [], + getPagePlayableDivs(page).map(x => findAll(".audio-sentence", x, true)) + ); +} - public pause() { - if (BloomPlayerCore.currentPlaybackMode === PlaybackMode.AudioPaused) { - return; - } - this.pausePlaying(); - this.startPause = new Date(); +export function pageHasAudio(page: HTMLElement): boolean { + return getPageAudioElements(page).length ? true : false; +} - // Note that neither music.pause() nor animations.PauseAnimations() check the state. - // If that changes, then this state setting will have to be moved to BloomPlayerCore. - BloomPlayerCore.currentPlaybackMode = PlaybackMode.AudioPaused; +export function play() { + if (currentPlaybackMode === PlaybackMode.AudioPlaying) { + return; // no change. } - - // This pauses the current player without setting the "AudioPaused" state or setting the - // startPause timestamp. If this method is called when resumption is possible, the calling - // method must take care of these values (as in the pause method directly above). - // Note that there's no "stop" method on player, only a "pause" method. This method is - // used both when "pausing" the narration while viewing a page and when stopping narration - // when changing pages. - private pausePlaying() { - const player = this.getPlayer(); - if (this.segments && this.segments.length && player) { - // Before reporting duration, try to check that we really are playing. - // a separate report is sent if play ends. - if (player.currentTime > 0 && !player.paused && !player.ended) { - this.reportPlayDuration(); - } - player.pause(); + setCurrentPlaybackMode(PlaybackMode.AudioPlaying); + // I'm not sure how getPlayer() can return null/undefined, but have seen it happen + // typically when doing something odd like trying to go back from the first page. + if (segments.length && getPlayer()) { + if (elementsToPlayConsecutivelyStack.length) { + handlePlayPromise(getPlayer().play()); + + // Resuming play. Only currentStartTime needs to be adjusted, but originalStartTime shouldn't be changed. + audioPlayCurrentStartTime = new Date().getTime(); + // in case we're resuming play, we need a new timout when the current subelement is finished + highlightNextSubElement(currentAudioSessionNum); + return; + } else { + // Pressing the play button in this case is triggering a replay of the current page, + // so we need to reset the highlighting. + playAllSentences(null); + return; } } + // Nothing real to play on this page, so PageNarrationComplete depends on a timeout. + // adjust startPlay by the elapsed pause. From this compute how much of durationOfPagesWithoutNarration + // remains. (I don't think this is quite right. It seems to always jump straight to the next page + // when resumed. But at least it is possible to pause on a page with no narration, which is better than + // the previous version.) + const pause = new Date().getTime() - startPause.getTime(); + startPlay = new Date(startPlay.getTime() + pause); + const remaining = + durationOfPagesWithoutNarration - + (new Date().getTime() - startPlay.getTime()); + if (remaining > 0) { + fakeNarrationTimer = window.setTimeout(() => { + setCurrentPlaybackMode(PlaybackMode.MediaFinished); + PageNarrationComplete?.raise(currentPlayPage!); + }, remaining); + } else { + // Somehow we already reached the limit. + PageNarrationComplete?.raise(currentPlayPage!); + } +} - public computeDuration(page: HTMLElement): void { - this.playerPage = page; - this.segments = this.getPageAudioElements(); - this.PageDuration = 0.0; - this.segmentIndex = -1; // so pre-increment in getNextSegment sets to 0. - this.startPlay = new Date(); - //console.log("started play at " + this.startPlay); - // in case we are already paused (but did manual advance), start computing - // the pause duration from the beginning of this page. - this.startPause = this.startPlay; - if (this.segments.length === 0) { - this.PageDuration = durationOfPagesWithoutNarration; - if (this.PageDurationAvailable) { - this.PageDurationAvailable.raise(page); - } - // Since there is nothing to play, we will never get an 'ended' event - // from the player. If we are going to advance pages automatically, - // we need to raise PageNarrationComplete some other way. - // A timeout allows us to raise it after the arbitrary duration we have - // selected. The tricky thing is to allow it to be paused. - clearTimeout(this.fakeNarrationTimer); - this.fakeNarrationTimer = window.setTimeout( - () => this.fakePageNarrationTimedOut(page), - this.PageDuration * 1000 - ); - this.fakeNarrationAborted = false; - return; - } - // trigger first duration evaluation. Each triggers another until we have them all. - this.getNextSegment(); - //this.getDurationPlayer().setAttribute("src", this.currentAudioUrl(this.segments[0].getAttribute("id"))); +export function pause() { + if (currentPlaybackMode === PlaybackMode.AudioPaused) { + return; } + pausePlaying(); + startPause = new Date(); - private getNextSegment() { - this.segmentIndex++; - if (this.segmentIndex < this.segments.length) { - const attrDuration = this.segments[this.segmentIndex].getAttribute( - "data-duration" - ); - if (attrDuration) { - // precomputed duration available, use it and go on. - this.PageDuration += parseFloat(attrDuration); - this.getNextSegment(); - return; - } - // Replace this with the commented code to have ask the browser for duration. - // (Also uncomment the getDurationPlayer method) - // However, this doesn't work in apps. - this.getNextSegment(); - // this.getDurationPlayer().setAttribute("src", - // this.currentAudioUrl(this.segments[this.segmentIndex].getAttribute("id"))); - } else { - if (this.PageDuration < durationOfPagesWithoutNarration) { - this.PageDuration = durationOfPagesWithoutNarration; - } - if (this.PageDurationAvailable) { - this.PageDurationAvailable.raise(this.playerPage); - } + // Note that neither music.pause() nor animations.PauseAnimations() check the state. + // If that changes, then this state setting might need attention. + setCurrentPlaybackMode(PlaybackMode.AudioPaused); +} + +// This pauses the current player without setting the "AudioPaused" state or setting the +// startPause timestamp. If this method is called when resumption is possible, the calling +// method must take care of these values (as in the pause method directly above). +// Note that there's no "stop" method on player, only a "pause" method. This method is +// used both when "pausing" the narration while viewing a page and when stopping narration +// when changing pages. +function pausePlaying() { + const player = getPlayer(); + // We're paused, so if we have a timer running to switch pages after a certain time, cancel it. + clearTimeout(fakeNarrationTimer); + if (segments && segments.length && player) { + // Before reporting duration, try to check that we really are playing. + // a separate report is sent if play ends. + if (player.currentTime > 0 && !player.paused && !player.ended) { + reportPlayDuration(); } + player.pause(); } +} - private fakePageNarrationTimedOut(page: HTMLElement) { - if (BloomPlayerCore.currentPlaybackMode === PlaybackMode.AudioPaused) { - this.fakeNarrationAborted = true; - clearTimeout(this.fakeNarrationTimer); - return; - } - // It's possible we experienced one or more pauses and therefore this timeout - // happened too soon. In that case, this.startPlay will have been adjusted by - // the pauses, so we can detect that here and start a new timeout which will - // occur at the appropriately delayed time. - const duration = - (new Date().getTime() - this.startPlay.getTime()) / 1000; - if (duration < this.PageDuration - 0.01) { - // too soon; try again. - clearTimeout(this.fakeNarrationTimer); - this.fakeNarrationTimer = window.setTimeout( - () => this.fakePageNarrationTimedOut(page), - (this.PageDuration - duration) * 1000 - ); - return; - } - if (this.PageNarrationComplete) { - this.PageNarrationComplete.raise(page); +// Figure out the total duration of the audio on the page. +// An earlier version of this code (see narration.ts around November 2023) +// was designed to run asnychronously so that if we don't have audio +// durations in the file, it would try to get the actual duration of the audio +// from the server. However, comments indicated that this approach did not +// work in mobile apps, and bloompubs have now long shipped with the durations. +// So I decided to simplify. +export function computeDuration(page: HTMLElement): number { + let pageDuration = 0.0; + getPageAudioElements(page).forEach(segment => { + const attrDuration = segment.getAttribute("data-duration"); + if (attrDuration) { + pageDuration += parseFloat(attrDuration); } + }); + if (pageDuration < durationOfPagesWithoutNarration) { + pageDuration = durationOfPagesWithoutNarration; } + return pageDuration; +} + +export function hidingPage() { + pausePlaying(); // Doesn't set AudioPaused state. Caller sets NewPage state. + clearTimeout(fakeNarrationTimer); +} - public hidingPage() { - this.pausePlaying(); // Doesn't set AudioPaused state. Caller sets NewPage state. +// Play the specified elements, one after the other. When the last completes (or at once if the array is empty), +// perform the 'then' action (typically used to play narration, which we put after videos). +// Todo: Bloom Player version, at least, should work with play/pause/resume/change page architecture. +export function playAllVideo(elements: HTMLVideoElement[], then: () => void) { + if (elements.length === 0) { + then(); + return; } + const video = elements[0]; + // Note: sometimes this event does not fire normally, even when the video is played to the end. + // I have not figured out why. It may be something to do with how we are trimming them. + // In Bloom, this is worked around by raising the ended event when we detect that it has paused past the end point + // in resetToStartAfterPlayingToEndPoint. + // In BloomPlayer, we may need to do something similar. Or possibly it's not a problem because export + // really shortens videos rather than just having BP limit the playback time. + video.addEventListener( + "ended", + () => { + playAllVideo(elements.slice(1), then); + }, + { once: true } + ); + // Review: do we need to do something to let the rest of the world know about this? + setCurrentPlaybackMode(PlaybackMode.VideoPlaying); + video.play(); } diff --git a/src/narrationUtils.ts b/src/narrationUtils.ts index e72540d1..e2b4de00 100644 --- a/src/narrationUtils.ts +++ b/src/narrationUtils.ts @@ -1,3 +1,5 @@ +import LiteEvent from "./event"; + export interface ISetHighlightParams { newElement: Element; shouldScrollToElement: boolean; @@ -5,6 +7,64 @@ export interface ISetHighlightParams { oldElement?: Element | null | undefined; // Optional. Provides some minor optimization if set. } +export enum PlaybackMode { + NewPage, // starting a new page ready to play + NewPageMediaPaused, // starting a new page in the "paused" state + VideoPlaying, // video is playing + VideoPaused, // video is paused + AudioPlaying, // narration and/or animation are playing (or possibly finished) + AudioPaused, // narration and/or animation are paused + MediaFinished // video, narration, and/or animation has played (possibly no media to play) + // Note that music can be playing when the state is either AudioPlaying or MediaFinished. +} + +export let currentPlaybackMode: PlaybackMode; +export function setCurrentPlaybackMode(mode: PlaybackMode) { + currentPlaybackMode = mode; +} + +// These functions support allowing a client (typically BloomPlayerCore) to register as +// the object that wants to receive notification of how long audio was played. +// Duration is in seconds. +export let durationReporter: (duration: number) => void; +export function listenForPlayDuration(reporter: (duration: number) => void) { + durationReporter = reporter; +} + +// A client may configure a function which can be called to find out whether a swipe +// is in progress...in BloomPlayerCore, this is implemented by a test on SWiper. +// It is currently only used when we need to scroll the content of a field we are +// playing. Bloom Desktop does not need to set it. +export let isSwipeInProgress: () => boolean; +export function setTestIsSwipeInProgress(tester: () => boolean) { + isSwipeInProgress = tester; +} + +// A client may configure a function which is passed the URL of each audio file narration plays. +export let logNarration: (src: string) => void; +export function setLogNarration(logger: (src: string) => void) { + logNarration = logger; +} + +let playerUrlPrefix = ""; +// In bloom player, figuring the url prefix is more complicated. We pass it in. +// In Bloom desktop, we don't call this at all. The code that would naturally do it +// is in the wrong iframe and it's a pain to get it to the right one. +// But there, the urlPrevix function works fine. +export function setPlayerUrlPrefix(prefix: string) { + playerUrlPrefix = prefix; +} + +export function urlPrefix(): string { + if (playerUrlPrefix) { + return playerUrlPrefix; + } + const bookSrc = window.location.href; + const index = bookSrc.lastIndexOf("/"); + const bookFolderUrl = bookSrc.substring(0, index); + return bookFolderUrl; +} + // We need to sort these by the tabindex of the containing bloom-translationGroup element. // We need a stable sort, which array.sort() does not provide: elements in the same // bloom-translationGroup, or where the translationGroup does not have a tabindex, @@ -49,3 +109,1077 @@ function getTgTabIndex(input: HTMLElement): string | null { } return tg.getAttribute("tabindex") || "999"; } + +// ------migrated from dragActivityNarration, common in narration.ts + +// Even though these can now encompass more than strict sentences, +// we continue to use this class name for backwards compatability reasons. +export const kAudioSentence = "audio-sentence"; +const kSegmentClass = "bloom-highlightSegment"; +const kImageDescriptionClass = "bloom-imageDescription"; +// Indicates that highlighting is briefly/temporarily suppressed, +// but may become highlighted later. +// For example, audio highlighting is suppressed until the related audio starts playing (to avoid flashes) +const kSuppressHighlightClass = "ui-suppressHighlight"; +// This event allows Narration to inform its controllers when we start/stop reading +// image descriptions. It is raised for each segment we read and passed true if the one +// we are about to read is an image description, false otherwise. +// Todo: wants a better name, it's not about toggling whether something is an image description, +// but about possibly updating the UI to reflect whether we are reading one. +export const ToggleImageDescription = new LiteEvent(); +const PageDurationAvailable = new LiteEvent; + +// On a typical page with narration, these are raised at the same time, when the last narration +// on the page finishes. But if there is no narration at all, PlayCompleted will be raised +// immediately (useful for example to disable a pause button), but PageNarrationComplete will +// be raised only after the standard delay for non-audio page (useful for auto-advancing to the next page). +export const PageNarrationComplete = new LiteEvent; +export const PlayCompleted = new LiteEvent; +// Raised when we can't play narration, specifically because the browser won't allow it until +// the user has interacted with the page. +export const PlayFailed = new LiteEvent; +let currentAudioId = ""; + +// The first one to play should be at the end for both of these +let elementsToPlayConsecutivelyStack: HTMLElement[] = []; +let subElementsWithTimings: Array<[Element, number]> = []; + +let startPause: Date; +let segments: HTMLElement[]; + +// A Session Number that keeps track of each time playAllAudio started. +// This might be needed to keep track of changing pages, or when we start new audio +// that will replace something already playing. +let currentAudioSessionNum: number = 0; + +// This represents the start time of the current playing of the audio. If the user presses pause/play, it will be reset. +// This is used for analytics reporting purposes +let audioPlayCurrentStartTime: number | null = null; // milliseconds (since 1970/01/01, from new Date().getTime()) + +let currentPlayPage: HTMLElement | null = null; +// Unused in Bloom desktop, but in Bloom player, current page might change while a series of sounds +// is playing. This lets us avoid starting the next sound if the page has changed in the meantime. +export function setCurrentPage(page: HTMLElement) { + currentPlayPage = page; +} +export function getCurrentPage(): HTMLElement { + return currentPlayPage!; +} + +function playCurrentInternal() { + if (currentPlaybackMode === PlaybackMode.AudioPlaying) { + let mediaPlayer = getPlayer(); + if (mediaPlayer) { + const element = getCurrentPage().querySelector( + `#${currentAudioId}` + ); + if (!element || !canPlayAudio(element)) { + playEnded(); + return; + } + + // I didn't comment this at the time, but my recollection is that making a new player + // each time helps with some cases where the old one was in a bad state, + // such as in the middle of pausing. + const src = mediaPlayer.getAttribute("src") ?? ""; + mediaPlayer.remove(); + mediaPlayer = getPlayer(); + mediaPlayer.setAttribute("src", src); + + // Regardless of whether we end up using timingsStr or not, + // we should reset this now in case the previous page used it and was still playing + // when the user flipped to the next page. + subElementsWithTimings = []; + + const timingsStr: string | null = element.getAttribute( + "data-audioRecordingEndTimes" + ); + if (timingsStr) { + const childSpanElements = element.querySelectorAll( + `span.${kSegmentClass}` + ); + const fields = timingsStr.split(" "); + const subElementCount = Math.min( + fields.length, + childSpanElements.length + ); + + for (let i = subElementCount - 1; i >= 0; --i) { + const durationSecs: number = Number(fields[i]); + if (isNaN(durationSecs)) { + continue; + } + subElementsWithTimings.push([ + childSpanElements.item(i), + durationSecs + ]); + } + } else { + // No timings string available. + // No need for us to do anything. The correct element is already highlighted by playAllSentences() (which needed to call setCurrent... anyway to set the audio player source). + // We'll just proceed along, start playing the audio, and playNextSubElement() will return immediately because there are no sub-elements in this case. + } + + const currentSegment = element as HTMLElement; + if (currentSegment) { + ToggleImageDescription.raise( + isImageDescriptionSegment(currentSegment) + ); + } + + gotErrorPlaying = false; + console.log("playing " + currentAudioId + "..." + mediaPlayer.src); + const promise = mediaPlayer.play(); + ++currentAudioSessionNum; + audioPlayCurrentStartTime = new Date().getTime(); + highlightNextSubElement(currentAudioSessionNum); + handlePlayPromise(promise); + } + } +} + +function isImageDescriptionSegment(segment: HTMLElement): boolean { + return segment.closest("." + kImageDescriptionClass) !== null; +} + +function canPlayAudio(current: Element): boolean { + return true; // currently no way to check +} + +// Moves the highlight to the next sub-element +// originalSessionNum: The value of currentAudioSessionNum at the time when the audio file started playing. +// This is used to check in the future if the timeouts we started are for the right session. +// startTimeInSecs is an optional fallback that will be used in case the currentTime cannot be determined from the audio player element. +function highlightNextSubElement( + originalSessionNum: number, + startTimeInSecs: number = 0 +) { + // the item should not be popped off the stack until it's completely done with. + const subElementCount = subElementsWithTimings.length; + + if (subElementCount <= 0) { + return; + } + + const topTuple = subElementsWithTimings[subElementCount - 1]; + const element = topTuple[0]; + const endTimeInSecs: number = topTuple[1]; + + setHighlightTo({ + newElement: element, + shouldScrollToElement: true, + disableHighlightIfNoAudio: false + }); + + const mediaPlayer: HTMLMediaElement = document.getElementById( + "bloom-audio-player" + )! as HTMLMediaElement; + let currentTimeInSecs: number = mediaPlayer.currentTime; + if (currentTimeInSecs <= 0) { + currentTimeInSecs = startTimeInSecs; + } + + // Handle cases where the currentTime has already exceeded the nextStartTime + // (might happen if you're unlucky in the thread queue... or if in debugger, etc.) + // But instead of setting time to 0, set the minimum highlight time threshold to 0.1 (this threshold is arbitrary). + const durationInSecs = Math.max(endTimeInSecs - currentTimeInSecs, 0.1); + + setTimeout(() => { + onSubElementHighlightTimeEnded(originalSessionNum); + }, durationInSecs * 1000); +} + +function handlePlayPromise(promise: Promise, player?: HTMLMediaElement) { + // In newer browsers, play() returns a promise which fails + // if the browser disobeys the command to play, as some do + // if the user hasn't 'interacted' with the page in some + // way that makes the browser think they are OK with it + // playing audio. In Gecko45, the return value is undefined, + // so we mustn't call catch. + if (promise && promise.catch) { + promise.catch((reason: any) => { + // There is an error handler here, but the HTMLMediaElement also has an error handler (which may end up calling playEnded()). + // In case it doesn't, we make sure here that it happens + handlePlayError(); + // This promise.catch error handler is the only one that handles NotAllowedException (that is, playback not started because user has not interacted with the page yet). + // However, older versions of browsers don't support promise from HTMLMediaElement.play(). So this cannot be the only error handler. + // Thus we need both the promise.catch error handler as well as the HTMLMediaElement's error handler. + // + // In many cases (such as NotSupportedError, which happens when the audio file isn't found), both error handlers will run. + // That is a little annoying but if the two don't conflict with each other it's not problematic. + + const playingWhat = player?.getAttribute("src") ?? "unknown"; + console.log( + "could not play sound: " + reason + " " + playingWhat + ); + + if ( + reason && + reason + .toString() + .includes( + "The play() request was interrupted by a call to pause()." + ) + ) { + // We were getting this error Aug 2020. I tried wrapping the line above which calls mediaPlayer.play() + // (currently `promise = mediaPlayer.play();`) in a setTimeout with 0ms. This seemed to fix the bug (with + // landscape books not having audio play initially -- BL-8887). But the root cause was actually that + // we ended up calling playAllSentences twice when the book first loaded. + // I fixed that in bloom-player-core. But I wanted to document the possible setTimeout fix here + // in case this issue ever comes up for a different reason. + console.log( + "See comment in narration.ts for possibly useful information regarding this error." + ); + } + + // Don't call removeAudioCurrent() here. The HTMLMediaElement's error handler will call playEnded() and calling removeAudioCurrent() here will mess up playEnded(). + // removeAudioCurrent(); + + // With some kinds of invalid sound file it keeps trying and plays over and over. + // But when we move on to play another sound, a pause here will mess things up. + // So instead I put a pause after we run out of sounds to try to play. + //getPlayer().pause(); + // if (Pause) { + // Pause.raise(); + // } + + // Get all the state (and UI) set correctly again. + // Not entirely sure about limiting this to NotAllowedError, but that's + // the one kind of play error that is fixed by the user just interacting. + // If there's some other reason we can't play, showing as paused may not + // be useful. See comments on the similar code in music.ts + if (reason.name === "NotAllowedError") { + PlayFailed.raise(); + } + }); + } +} + +// Handles a timeout indicating that the expected time for highlighting the current subElement has ended. +// If we've really played to the end of that subElement, highlight the next one (if any). +// originalSessionNum: The value of currentAudioSessionNum at the time when the audio file started playing. +// This is used to check in the future if the timeouts we started are for the right session +function onSubElementHighlightTimeEnded(originalSessionNum: number) { + // Check if the user has changed pages since the original audio for this started playing. + // Note: Using the timestamp allows us to detect switching to the next page and then back to this page. + // Using playerPage (HTMLElement) does not detect that. + if (originalSessionNum !== currentAudioSessionNum) { + return; + } + // Seems to be needed to prevent jumping to the next subelement when not permitted to play by browser. + // Not sure why the check below on mediaPlayer.currentTime does not prevent this. + if (currentPlaybackMode === PlaybackMode.AudioPaused) { + return; + } + + const subElementCount = subElementsWithTimings.length; + if (subElementCount <= 0) { + return; + } + + const mediaPlayer: HTMLMediaElement = document.getElementById( + "bloom-audio-player" + )! as HTMLMediaElement; + if (mediaPlayer.ended || mediaPlayer.error) { + // audio playback ended. No need to highlight anything else. + // (No real need to remove the highlights either, because playEnded() is supposed to take care of that.) + return; + } + const playedDurationInSecs: number | undefined | null = + mediaPlayer.currentTime; + + // Peek at the next sentence and see if we're ready to start that one. (We might not be ready to play the next audio if the current audio got paused). + const subElementWithTiming = subElementsWithTimings[subElementCount - 1]; + const nextStartTimeInSecs = subElementWithTiming[1]; + + if (playedDurationInSecs && playedDurationInSecs < nextStartTimeInSecs) { + // Still need to wait. Exit this function early and re-check later. + const minRemainingDurationInSecs = + nextStartTimeInSecs - playedDurationInSecs; + setTimeout(() => { + onSubElementHighlightTimeEnded(originalSessionNum); + }, minRemainingDurationInSecs * 1000); + + return; + } + + subElementsWithTimings.pop(); + + highlightNextSubElement(originalSessionNum, nextStartTimeInSecs); +} + +function setSoundFrom(element: Element) { + const firstAudioSentence = getFirstAudioSentenceWithinElement(element); + const id: string = firstAudioSentence ? firstAudioSentence.id : element.id; + setCurrentAudioId(id); +} + +function setCurrentAudioId(id: string) { + if (!currentAudioId || currentAudioId !== id) { + currentAudioId = id; + updatePlayerStatus(); + } +} + +function updatePlayerStatus() { + const player = getPlayer(); + if (!player) { + return; + } + // Any time we change the src, the player will pause. + // So if we're playing currently, we'd better report whatever time + // we played. + if (player.currentTime > 0 && !player.paused && !player.ended) { + reportPlayDuration(); + } + const url = currentAudioUrl(currentAudioId); + logNarration(url); + player.setAttribute("src", url + "?nocache=" + new Date().getTime()); +} + +function getPlayer(): HTMLMediaElement { + const audio = getAudio("bloom-audio-player", _ => {}); + // We used to do this in the init call, but sometimes the function didn't get called. + // Suspecting that there are cases, maybe just in storybook, where a new instance + // of the narration object gets created, but the old audio element still exists. + // Make sure the current instance has our end function. + // Because it is a fixed function for the lifetime of this object, addEventListener + // will not add it repeatedly. + audio.addEventListener("ended", playEnded); + audio.addEventListener("error", handlePlayError); + // If we are suppressing hiliting something until we confirm that the audio really exists, + // we can stop doing so: the audio is playing. + audio.addEventListener("playing", removeHighlightSuppression); + return audio; +} +function removeHighlightSuppression() { + Array.from( + document.getElementsByClassName(kSuppressHighlightClass) + ).forEach(newElement => + newElement.classList.remove(kSuppressHighlightClass) + ); +} + +function currentAudioUrl(id: string): string { + const result = urlPrefix() + "/audio/" + id + ".mp3"; + console.log("trying to play " + result); + return result; +} + +function getAudio(id: string, init: (audio: HTMLAudioElement) => void) { + let player: HTMLAudioElement | null = document.querySelector( + "#" + id + ) as HTMLAudioElement; + if (player && !player.play) { + player.remove(); + player = null; + } + if (!player) { + player = document.createElement("audio") as HTMLAudioElement; + player.setAttribute("id", id); + document.body.appendChild(player); + init(player); + } + return player as HTMLMediaElement; +} + +function playEnded(): void { + // Not sure if this is necessary, since both 'playCurrentInternal()' and 'reportPlayEnded()' + // will toggle image description already, but if we've just gotten to the end of our "stack", + // it may be needed. + if (ToggleImageDescription) { + ToggleImageDescription.raise(false); + } + reportPlayDuration(); + if ( + elementsToPlayConsecutivelyStack && + elementsToPlayConsecutivelyStack.length > 0 + ) { + elementsToPlayConsecutivelyStack.pop(); // get rid of the last one we played + const newStackCount = elementsToPlayConsecutivelyStack.length; + if (newStackCount > 0) { + // More items to play + const nextElement = + elementsToPlayConsecutivelyStack[newStackCount - 1]; + setSoundAndHighlight(nextElement, true); + playCurrentInternal(); + } else { + reportPlayEnded(); + removeAudioCurrent(); + // In some error conditions, we need to stop repeating attempts to play. + getPlayer().pause(); + } + } +} + +function reportPlayEnded() { + elementsToPlayConsecutivelyStack = []; + subElementsWithTimings = []; + + removeAudioCurrent(); + PageNarrationComplete.raise(currentPlayPage!); + PlayCompleted.raise(); +} + +function reportPlayDuration() { + if (!audioPlayCurrentStartTime || !durationReporter) { + return; + } + const currentTime = new Date().getTime(); + const duration = (currentTime - audioPlayCurrentStartTime) / 1000; + durationReporter(duration); +} + +function setSoundAndHighlight( + newElement: Element, + disableHighlightIfNoAudio: boolean, + oldElement?: Element | null | undefined +) { + setHighlightTo({ + newElement, + shouldScrollToElement: true, // Always true in bloom-player version + disableHighlightIfNoAudio, + oldElement + }); + setSoundFrom(newElement); +} + +function setHighlightTo({ + newElement, + shouldScrollToElement, + disableHighlightIfNoAudio, + oldElement +}: ISetHighlightParams) { + // This should happen even if oldElement and newElement are the same. + if (shouldScrollToElement) { + // Wrap it in a try/catch so that if something breaks with this minor/nice-to-have feature of scrolling, + // the main responsibilities of this method can still proceed + try { + scrollElementIntoView(newElement); + } catch (e) { + console.error(e); + } + } + + if (oldElement === newElement) { + // No need to do much, and better not to, so that we can avoid any temporary flashes as the highlight is removed and re-applied + return; + } + + removeAudioCurrent(); + + if (disableHighlightIfNoAudio) { + const mediaPlayer = getPlayer(); + const isAlreadyPlaying = mediaPlayer.currentTime > 0; + + // If it's already playing, no need to disable (Especially in the Soft Split case, where only one file is playing but multiple sentences need to be highlighted). + if (!isAlreadyPlaying) { + // Start off in a highlight-disabled state so we don't display any momentary highlight for cases where there is no audio for this element. + // In react-based bloom-player, canPlayAudio() can't trivially identify whether or not audio exists, + // so we need to incorporate a derivative of Bloom Desktop's .ui-suppressHighlight code + newElement.classList.add(kSuppressHighlightClass); + } + } + + newElement.classList.add("ui-audioCurrent"); + // If the current audio is part of a (currently typically hidden) image description, + // highlight the image. + // it's important to check for imageDescription on the translationGroup; + // we don't want to highlight the image while, for example, playing a TOP box content. + const translationGroup = newElement.closest(".bloom-translationGroup"); + if ( + translationGroup && + translationGroup.classList.contains(kImageDescriptionClass) + ) { + const imgContainer = translationGroup.closest(".bloom-imageContainer"); + if (imgContainer) { + imgContainer.classList.add("ui-audioCurrentImg"); + } + } +} + +// Removes the .ui-audioCurrent class from all elements (also ui-audioCurrentImg) +// Equivalent of removeAudioCurrentFromPageDocBody() in BloomDesktop. +function removeAudioCurrent() { + // Note that HTMLCollectionOf's length can change if you change the number of elements matching the selector. + const audioCurrentCollection: HTMLCollectionOf = document.getElementsByClassName( + "ui-audioCurrent" + ); + + // Convert to an array whose length won't be changed + const audioCurrentArray: Element[] = Array.from(audioCurrentCollection); + + for (let i = 0; i < audioCurrentArray.length; i++) { + audioCurrentArray[i].classList.remove("ui-audioCurrent"); + } + const currentImg = document.getElementsByClassName("ui-audioCurrentImg")[0]; + if (currentImg) { + currentImg.classList.remove("ui-audioCurrentImg"); + } +} + +// Scrolls an element into view. +function scrollElementIntoView(element: Element) { + // In Bloom Player, scrollIntoView can interfere with page swipes, + // so Bloom Player needs some smarts about when to call it... + if (isSwipeInProgress?.()) { + // This alternative implementation doesn't use scrollIntoView (Which interferes with swiper). + // Since swiping is only active at the beginning (usually while the 1st element is playing) + // it should generally be good enough just to reset the scroll of the scroll parent to the top. + + // Assumption: Assumes the editable is the scrollbox. + // If this is not the case, you can use JQuery's scrollParent() function or other equivalent + const scrollAncestor = getEditable(element); + if (scrollAncestor) { + scrollAncestor.scrollTop = 0; + } + return; + } + + let mover = element as HTMLElement; // by default make the element itself scrollIntoView + if (window.getComputedStyle(element.parentElement!).position !== "static") { + // We can make a new element absolutely positioned and it will be relative to the parent. + // The idea is to make an element much narrower than the element we are + // trying to make visible, since we don't want horizontal movement. Quite possibly, + // as in BL-11038, only some white space is actually off-screen. But even if the author + // has positioned a bubble so some text is cut off, we don't want horizontal scrolling, + // which inside swiper will weirdly pull in part of the next page. + // (In the pathological case that the bubble is more than half hidden, we'll do the + // horizontal scroll, despite the ugliness of possibly showing part of the next page.) + // Note that elt may be a span, when scrolling chunks of text into view to play. + // I thought about using scrollWidth/Height to include any part of the element + // that is scrolled out of view, but for some reason these are always zero for spans. + // OffsetHeight seems to give the full height, though docs seem to indicate that it + // should not include invisible areas. + const elt = element as HTMLElement; + mover = document.createElement("div"); + mover.style.position = "absolute"; + mover.style.top = elt.offsetTop + "px"; + + // now we need what for a block would be offsetLeft. However, for a span, that + // yields the offset of the top left corner, which may be in the middle + // of a line. + const bounds = elt.getBoundingClientRect(); + const parent = elt.parentElement; + const parentBounds = parent?.getBoundingClientRect(); + const scale = parentBounds!.width / parent!.offsetWidth; + const leftRelativeToParent = (bounds.left - parentBounds!.left) / scale; + + mover.style.left = leftRelativeToParent + elt.offsetWidth / 2 + "px"; + mover.style.height = elt.offsetHeight + "px"; + mover.style.width = "0"; + element.parentElement?.insertBefore(mover, element); + } + + mover.scrollIntoView({ + // Animated instead of sudden + behavior: "smooth", + + // "nearest" setting does lots of smarts for us (compared to us deciding when to use "start" or "end") + // Seems to reduce unnecessary scrolling compared to start (aka true) or end (aka false). + // Refer to https://drafts.csswg.org/cssom-view/#scroll-an-element-into-view, + // which seems to imply that it won't do any scrolling if the two relevant edges are already inside. + block: "nearest" + + // horizontal alignment is controlled by "inline". We'll leave it as its default ("nearest") + // which typically won't move things at all horizontally + }); + if (mover !== element) { + mover.parentElement?.removeChild(mover); + } +} + +function getEditable(element: Element): Element | null { + if (element.classList.contains("bloom-editable")) { + return element; + } else { + return element.closest(".bloom-editable"); // Might be null + } +} + +// If something goes wrong playing a media element, typically that we don't actually have a recording +// for a particular one, we seem to sometimes get an error event, while other times, the promise returned +// by play() is rejected. Both cases call handlePlayError, which calls playEnded, but in case we get both, +// we don't want to call playEnded twice. +let gotErrorPlaying = false; + +function handlePlayError() { + if (gotErrorPlaying) { + console.log("Already got error playing, not handling again"); + return; + } + gotErrorPlaying = true; + console.log("Error playing, handling"); + setTimeout(() => { + playEnded(); + }, 100); +} + +function getFirstAudioSentenceWithinElement( + element: Element | null +): Element | null { + const audioSentences = getAudioSegmentsWithinElement(element); + if (!audioSentences || audioSentences.length === 0) { + return null; + } + + return audioSentences[0]; +} + +function getAudioSegmentsWithinElement(element: Element | null): Element[] { + const audioSegments: Element[] = []; + + if (element) { + if (element.classList.contains(kAudioSentence)) { + audioSegments.push(element); + } else { + const collection = element.getElementsByClassName(kAudioSentence); + for (let i = 0; i < collection.length; ++i) { + const audioSentenceElement = collection.item(i); + if (audioSentenceElement) { + audioSegments.push(audioSentenceElement); + } + } + } + } + + return audioSegments; +} + +// --------- migrated from narration.ts, not in dragActivityNarration + +let durationOfPagesWithoutNarration = 3.0; // seconds +export function setDurationOfPagesWithoutNarration(d: number) { + durationOfPagesWithoutNarration = d; +} +let includeImageDescriptions: boolean = true; +export function setIncludeImageDescriptions(b: boolean) { + includeImageDescriptions = b; +} +let startPlay: Date; +let fakeNarrationAborted: boolean = false; +let fakeNarrationTimer: number; +export let PageDuration: number; + +export function pause() { + if (currentPlaybackMode === PlaybackMode.AudioPaused) { + return; + } + pausePlaying(); + startPause = new Date(); + + // Note that neither music.pause() nor animations.PauseAnimations() check the state. + // If that changes, then this state setting might need attention. + setCurrentPlaybackMode(PlaybackMode.AudioPaused); +} + +// This pauses the current player without setting the "AudioPaused" state or setting the +// startPause timestamp. If this method is called when resumption is possible, the calling +// method must take care of these values (as in the pause method directly above). +// Note that there's no "stop" method on player, only a "pause" method. This method is +// used both when "pausing" the narration while viewing a page and when stopping narration +// when changing pages. +function pausePlaying() { + const player = getPlayer(); + if (segments && segments.length && player) { + // Before reporting duration, try to check that we really are playing. + // a separate report is sent if play ends. + if (player.currentTime > 0 && !player.paused && !player.ended) { + reportPlayDuration(); + } + player.pause(); + } +} + +export function hidingPage() { + pausePlaying(); // Doesn't set AudioPaused state. Caller sets NewPage state. + clearTimeout(pageNarrationCompleteTimer); +} + + // Roughly equivalent to BloomDesktop's AudioRecording::listen() function. + // As long as there is audio on the page, this method will play it. + export function playAllSentences(page: HTMLElement | null): void { + if (!page && !currentPlayPage) { + return; // this shouldn't happen + } + if (page) { + currentPlayPage = page; // Review: possibly redundant? Do all callers set currentPlayPage independently? + } + const mediaPlayer = getPlayer(); + if (mediaPlayer) { + mediaPlayer.pause(); + mediaPlayer.currentTime = 0; + } + + // Invalidate old ID, even if there's no new audio to play. + // (Deals with the case where you are on a page with audio, switch to a page without audio, then switch back to original page) + ++currentAudioSessionNum; + + fixHighlighting(); + + // Sorted into the order we want to play them, then reversed so we + // can more conveniently pop the next one to play from the end of the stack. + elementsToPlayConsecutivelyStack = sortAudioElements( + getPageAudioElements() + ).reverse(); + + const stackSize = elementsToPlayConsecutivelyStack.length; + if (stackSize === 0) { + // Nothing to play. Wait the standard amount of time anyway, in case we're autoadvancing. + if (PageNarrationComplete) { + pageNarrationCompleteTimer = window.setTimeout(() => { + PageNarrationComplete.raise(); + }, durationOfPagesWithoutNarration * 1000); + } + if (PlayCompleted) { + PlayCompleted.raise(); + } + return; + } + + const firstElementToPlay = elementsToPlayConsecutivelyStack[ + stackSize - 1 + ]; // Remember to pop it when you're done playing it. (i.e., in playEnded) + + setSoundAndHighlight(firstElementToPlay, true); + playCurrentInternal(); + return; + } + + let pageNarrationCompleteTimer: number; +// Indicates that the element should be highlighted. +const kEnableHighlightClass = "ui-enableHighlight"; +// Indicates that the element should NOT be highlighted. +// For example, some elements have highlighting prevented at this level +// because its content has been broken into child elements, only some of which show the highlight +const kDisableHighlightClass = "ui-disableHighlight"; +// Match space or   (\u00a0). Must have three or more in a row to match. +// Note: Multi whitespace text probably contains a bunch of   followed by a single normal space at the end. +const multiSpaceRegex = /[ \u00a0]{3,}/; +const multiSpaceRegexGlobal = new RegExp(multiSpaceRegex, "g"); +/** + * Finds and fixes any elements on the page that should have their audio-highlighting disabled. + */ +function fixHighlighting() { + // Note: Only relevant when playing by sentence (but note, this can make Record by Text Box -> Split or Record by Sentence, Play by Sentence) + // Play by Text Box highlights the whole paragraph and none of this really matters. + // (the span selector won't match anyway) + const audioElements = getPageAudioElements(); + audioElements.forEach(audioElement => { + // FYI, don't need to process the bloom-linebreak spans. Nothing bad happens, just unnecessary. + const matches = findAll( + "span:not(.bloom-linebreak)", + audioElement, + true + ); + matches.forEach(element => { + // Simple check to help ensure that elements that don't need to be modified will remain untouched. + // This doesn't consider whether text that shouldn't be highlighted is already in inside an + // element with highlight disabled, but that's ok. The code down the stack checks that. + const containsNonHighlightText = !!element.innerText.match( + multiSpaceRegex + ); + + if (containsNonHighlightText) { + fixHighlightingInNode(element, element); + } + }); + }); +} + +/** + * Recursively fixes the audio-highlighting within a node (whether element node or text node) + * @param node The node to recursively fix + * @param startingSpan The starting span, AKA the one that will receive .ui-audioCurrent in the future. + */ +function fixHighlightingInNode(node: Node, startingSpan: HTMLSpanElement) { + if ( + node.nodeType === Node.ELEMENT_NODE && + (node as Element).classList.contains(kDisableHighlightClass) + ) { + // No need to process bloom-highlightDisabled elements (they've already been processed) + return; + } else if (node.nodeType === Node.TEXT_NODE) { + // Leaf node. Fix the highlighting, then go back up the stack. + fixHighlightingInTextNode(node, startingSpan); + return; + } else { + // Recursive case + const childNodesCopy = Array.from(node.childNodes); // Make a copy because node.childNodes is being mutated + childNodesCopy.forEach(childNode => { + fixHighlightingInNode(childNode, startingSpan); + }); + } +} + +/** + * Analyzes a text node and fixes its highlighting. + */ +function fixHighlightingInTextNode( + textNode: Node, + startingSpan: HTMLSpanElement +) { + if (textNode.nodeType !== Node.TEXT_NODE) { + throw new Error( + "Invalid argument to fixMultiSpaceInTextNode: node must be a TextNode" + ); + } + + if (!textNode.nodeValue) { + return; + } + + // string.matchAll would be cleaner, but not supported in all browsers (in particular, FF60) + // Use RegExp.exec for greater compatibility. + multiSpaceRegexGlobal.lastIndex = 0; // RegExp.exec is stateful! Need to reset the state. + const matches: { + text: string; + startIndex: number; + endIndex: number; // the index of the first character to exclude + }[] = []; + let regexResult: RegExpExecArray | null; + while ( + (regexResult = multiSpaceRegexGlobal.exec( + textNode.nodeValue + )) != null + ) { + regexResult.forEach(matchingText => { + matches.push({ + text: matchingText, + startIndex: + multiSpaceRegexGlobal.lastIndex - + matchingText.length, + endIndex: multiSpaceRegexGlobal.lastIndex // the index of the first character to exclude + }); + }); + } + + // First, generate the new DOM elements with the fixed highlighting. + const newNodes: Node[] = []; + if (matches.length === 0) { + // No matches + newNodes.push(makeHighlightedSpan(textNode.nodeValue)); + } else { + let lastMatchEndIndex = 0; // the index of the first character to exclude of the last match + for (let i = 0; i < matches.length; ++i) { + const match = matches[i]; + + const preMatchText = textNode.nodeValue.slice( + lastMatchEndIndex, + match.startIndex + ); + lastMatchEndIndex = match.endIndex; + newNodes.push(makeHighlightedSpan(preMatchText)); + + newNodes.push(document.createTextNode(match.text)); + + if (i === matches.length - 1) { + const postMatchText = textNode.nodeValue.slice( + match.endIndex + ); + if (postMatchText) { + newNodes.push(makeHighlightedSpan(postMatchText)); + } + } + } + } + + // Next, replace the old DOM element with the new DOM elements + const oldNode = textNode; + if (oldNode.parentNode && newNodes && newNodes.length > 0) { + for (let i = 0; i < newNodes.length; ++i) { + const nodeToInsert = newNodes[i]; + oldNode.parentNode.insertBefore(nodeToInsert, oldNode); + } + + oldNode.parentNode.removeChild(oldNode); + + // We need to set ancestor's background back to transparent (instead of highlighted), + // and let each of the newNodes's styles control whether to be highlighted or transparent. + // If ancestor was highlighted but one of its new descendant nodes was transparent, + // all that would happen is the descendant would allow the ancestor's highlight color to show through, + // which doesn't achieve what we want :( + startingSpan.classList.add(kDisableHighlightClass); + } +} + +function makeHighlightedSpan(textContent: string) { + const newSpan = document.createElement("span"); + newSpan.classList.add(kEnableHighlightClass); + newSpan.appendChild(document.createTextNode(textContent)); + return newSpan; +} + + // Optional param is for use when 'playerPage' has NOT been initialized. +// Not using the optional param assumes 'playerPage' has been initialized +function getPageAudioElements(page?: HTMLElement): HTMLElement[] { + return [].concat.apply( + [], + getPagePlayableDivs(page).map(x => + findAll(".audio-sentence", x, true) + ) + ); +} + + // Returns all elements that match CSS selector {expr} as an array. +// Querying can optionally be restricted to {container}’s descendants +// If includeSelf is true, it includes both itself as well as its descendants. +// Otherwise, it only includes descendants. +// Also filters out imageDescriptions if we aren't supposed to be reading them. +function findAll( + expr: string, + container: HTMLElement, + includeSelf: boolean = false +): HTMLElement[] { + // querySelectorAll checks all the descendants + const allMatches: HTMLElement[] = [].slice.call( + (container || document).querySelectorAll(expr) + ); + + // Now check itself + if (includeSelf && container && container.matches(expr)) { + allMatches.push(container); + } + + return includeImageDescriptions + ? allMatches + : allMatches.filter( + match => !isImageDescriptionSegment(match) + ); +} + +function getPlayableDivs(container: HTMLElement) { + // We want to play any audio we have from divs the user can see. + // This is a crude test, but currently we always use display:none to hide unwanted languages. + return findAll(".bloom-editable", container).filter( + e => window.getComputedStyle(e).display !== "none" + ); +} + +// Optional param is for use when 'playerPage' has NOT been initialized. +// Not using the optional param assumes 'playerPage' has been initialized +function getPagePlayableDivs(page?: HTMLElement): HTMLElement[] { + return getPlayableDivs(page ? page : currentPlayPage!); +} + +export function play() { + if (currentPlaybackMode === PlaybackMode.AudioPlaying) { + return; // no change. + } + setCurrentPlaybackMode(PlaybackMode.AudioPlaying); + // I'm not sure how getPlayer() can return null/undefined, but have seen it happen + // typically when doing something odd like trying to go back from the first page. + if (segments.length && getPlayer()) { + if (elementsToPlayConsecutivelyStack.length) { + handlePlayPromise(getPlayer().play()); + + // Resuming play. Only currentStartTime needs to be adjusted, but originalStartTime shouldn't be changed. + audioPlayCurrentStartTime = new Date().getTime(); + } else { + // Pressing the play button in this case is triggering a replay of the current page, + // so we need to reset the highlighting. + playAllSentences(null); + return; + } + } + // adjust startPlay by the elapsed pause. This will cause fakePageNarrationTimedOut to + // start a new timeout if we are depending on it to fake PageNarrationComplete. + const pause = new Date().getTime() - startPause.getTime(); + startPlay = new Date(startPlay.getTime() + pause); + //console.log("paused for " + pause + " and adjusted start time to " + startPlay); + if (fakeNarrationAborted) { + // we already paused through the timeout for normal advance. + // This call (now we are not paused and have adjusted startPlay) + // will typically start a new timeout. If we are very close to + // the desired duration it may just raise the event at once. + // Either way we should get the event raised exactly once + // at very close to the right time, allowing for pauses. + fakeNarrationAborted = false; + fakePageNarrationTimedOut(currentPlayPage!); + } + // in case we're resuming play, we need a new timout when the current subelement is finished + highlightNextSubElement(currentAudioSessionNum); +} + + +function fakePageNarrationTimedOut(page: HTMLElement) { + if (currentPlaybackMode === PlaybackMode.AudioPaused) { + fakeNarrationAborted = true; + clearTimeout(fakeNarrationTimer); + return; + } + // It's possible we experienced one or more pauses and therefore this timeout + // happened too soon. In that case, startPlay will have been adjusted by + // the pauses, so we can detect that here and start a new timeout which will + // occur at the appropriately delayed time. + const duration = + (new Date().getTime() - startPlay.getTime()) / 1000; + if (duration < PageDuration - 0.01) { + // too soon; try again. + clearTimeout(fakeNarrationTimer); + fakeNarrationTimer = window.setTimeout( + () => fakePageNarrationTimedOut(page), + (PageDuration - duration) * 1000 + ); + return; + } + if (PageNarrationComplete) { + PageNarrationComplete.raise(page); + } +} + +// Figure out the total duration of the audio on the page. +// Currently has side effects of setting the current page and segments. +// I think that should be removed. +// An earlier version of this code (see narration.ts around November 2023) +// was designed to run asnychronously so that if we don't have audio +// durations in the file, it would try to get the actual duration of the audio +// from the server. However, comments indicated that this approach did not +// work in mobile apps, and bloompubs have now long shipped with the durations. +// So I decided to simplify. +export function computeDuration(page: HTMLElement): number { + currentPlayPage = page; + segments = getPageAudioElements(); + PageDuration = 0.0; + startPlay = new Date(); + //console.log("started play at " + startPlay); + // in case we are already paused (but did manual advance), start computing + // the pause duration from the beginning of this page. + startPause = startPlay; + if (segments.length === 0) { + PageDuration = durationOfPagesWithoutNarration; + if (PageDurationAvailable) { + PageDurationAvailable.raise(page); + } + // Since there is nothing to play, we will never get an 'ended' event + // from the player. If we are going to advance pages automatically, + // we need to raise PageNarrationComplete some other way. + // A timeout allows us to raise it after the arbitrary duration we have + // selected. The tricky thing is to allow it to be paused. + clearTimeout(fakeNarrationTimer); + fakeNarrationTimer = window.setTimeout( + () => fakePageNarrationTimedOut(page), + PageDuration * 1000 + ); + fakeNarrationAborted = false; + return PageDuration; + } + + segments.forEach((segment, index) => { + const attrDuration = segment.getAttribute( + "data-duration" + ); + if (attrDuration) { + // precomputed duration available, use it and go on. + PageDuration += parseFloat(attrDuration); + } + }); + if (PageDuration < durationOfPagesWithoutNarration) { + PageDuration = durationOfPagesWithoutNarration; + } + return PageDuration; +} + +export function pageHasAudio(page: HTMLElement): boolean { + return getPageAudioElements(page).length ? true : false; +} diff --git a/src/stories/index.tsx b/src/stories/index.tsx index bfaafdb0..3c3fd0e0 100644 --- a/src/stories/index.tsx +++ b/src/stories/index.tsx @@ -200,6 +200,10 @@ AddBloomPlayerStory( "Activity/Choice activities from Bloom 5.4", "testbooks/Bloom5.4-activities/Bloom5.4-activities.htm" ); +AddBloomPlayerStory( + "Activity/Dragging", + "testbooks/Word%20Slider/Word%20Slider.htm" +); AddBloomPlayerStory( "Book with two audio sentences on cover", "https://s3.amazonaws.com/bloomharvest/namitaj%40chetana.org.in/78c7e561-ce24-4e5d-ad0a-6af141d9d0af/bloomdigital%2findex.htm" diff --git a/src/video.ts b/src/video.ts index 15a3f3c9..540e7e48 100644 --- a/src/video.ts +++ b/src/video.ts @@ -1,6 +1,11 @@ import LiteEvent from "./event"; -import { BloomPlayerCore, PlaybackMode } from "./bloom-player-core"; +import { BloomPlayerCore } from "./bloom-player-core"; import { isMacOrIOS } from "./utilities/osUtils"; +import { + currentPlaybackMode, + setCurrentPlaybackMode, + PlaybackMode +} from "./narration"; // class Video contains functionality to get videos to play properly in bloom-player @@ -65,7 +70,7 @@ export class Video { }); } }; - if (BloomPlayerCore.currentPlaybackMode === PlaybackMode.VideoPaused) { + if (currentPlaybackMode === PlaybackMode.VideoPaused) { this.currentVideoElement.pause(); } else { const videoElement = this.currentVideoElement; @@ -134,14 +139,14 @@ export class Video { } public play() { - if (BloomPlayerCore.currentPlaybackMode == PlaybackMode.VideoPlaying) { + if (currentPlaybackMode === PlaybackMode.VideoPlaying) { return; // no change. } const videoElement = this.currentVideoElement; if (!videoElement) { return; // no change } - BloomPlayerCore.currentPlaybackMode = PlaybackMode.VideoPlaying; + setCurrentPlaybackMode(PlaybackMode.VideoPlaying); // If it has ended, it's going to replay from the beginning, even though // (to prevent an abrupt visual effect) we didn't reset currentTime when it ended. this.videoStartTime = this.videoEnded ? 0 : videoElement.currentTime; @@ -149,11 +154,11 @@ export class Video { } public pause() { - if (BloomPlayerCore.currentPlaybackMode == PlaybackMode.VideoPaused) { + if (currentPlaybackMode == PlaybackMode.VideoPaused) { return; } this.pauseCurrentVideo(); - BloomPlayerCore.currentPlaybackMode = PlaybackMode.VideoPaused; + setCurrentPlaybackMode(PlaybackMode.VideoPaused); } private pauseCurrentVideo() { From 0ab92eabfb04f5f933d014d044b1ed00d6347963 Mon Sep 17 00:00:00 2001 From: John Thomson Date: Thu, 18 Jul 2024 12:24:52 -0500 Subject: [PATCH 2/2] Improving narration, adding data-sound, fixing bugs, cleaning up --- package.json | 1 - src/activities/ActivityContext.ts | 20 +- src/activities/activityManager.ts | 42 +- .../dragActivities/DragToDestination.ts | 33 +- src/bloom-player-core.tsx | 43 +- src/dragActivityRuntime.ts | 147 +- src/event.ts | 4 +- src/narration.ts | 100 +- src/narrationUtils.ts | 1185 ----------------- src/stories/index.tsx | 4 - 10 files changed, 259 insertions(+), 1320 deletions(-) delete mode 100644 src/narrationUtils.ts diff --git a/package.json b/package.json index 2b21fa9c..bc3d48ae 100644 --- a/package.json +++ b/package.json @@ -110,7 +110,6 @@ "dist/*.css", "dist/*.tsv" ], - "packageManager": "yarn@1.22.19", "volta": { "node": "16.14.0", "yarn": "1.22.19" diff --git a/src/activities/ActivityContext.ts b/src/activities/ActivityContext.ts index 740887b6..4ec78f8a 100644 --- a/src/activities/ActivityContext.ts +++ b/src/activities/ActivityContext.ts @@ -75,11 +75,13 @@ export class ActivityContext { // NB: if this stops working in storybook; the file should be found because the package.json // script that starts storybook has a "--static-dir" option that should include the folder // containing the standard activity sounds. - this.playSound(rightAnswer); + // require on an mp3 gives us some sort of module object where the url of the sound is its 'default' + this.playSound(rightAnswer.default); } public playWrong() { - this.playSound(wrongAnswer); + // require on an mp3 gives us some sort of module object where the url of the sound is its 'default' + this.playSound(wrongAnswer.default); } private getPagePlayer(): any { @@ -96,9 +98,9 @@ export class ActivityContext { return player; } - public playSound(url) { + public playSound(url: string) { const player = this.getPagePlayer(); - player.setAttribute("src", url.default); + player.setAttribute("src", url); player.play(); } @@ -138,16 +140,6 @@ export class ActivityContext { target.addEventListener(name, listener, options); } - public removeEventListener( - name: string, - target: Element, - listener: EventListener, - options?: AddEventListenerOptions | undefined - ) { - // we could try to remove it from this.listeners, but it's harmless to remove it again - target.removeEventListener(name, listener, options); - } - // this is called by the activity manager after it stops the activity. public stop() { // detach all the listeners diff --git a/src/activities/activityManager.ts b/src/activities/activityManager.ts index 4d464e3a..35c97d24 100644 --- a/src/activities/activityManager.ts +++ b/src/activities/activityManager.ts @@ -24,6 +24,10 @@ export interface IActivityObject { // This is acting on the real DOM, so this is the time to set up event handlers, etc. showingPage: (context: ActivityContext) => void; + // This is called in place of the normal code that plays sound and animations when the page first appears, + // if the activity's requirements specify soundManagement: true. + doInitialSoundAndAnimation?: (context: ActivityContext) => void; + stop: () => void; } // Constructing stuff from interfaces has problems with typescript at the moment. @@ -36,7 +40,10 @@ export interface IActivityRequirements { dragging?: boolean; clicking?: boolean; typing?: boolean; - soundManagement?: boolean; // suppress normal sound (and music, and animation) + // suppress normal sound (and music, and animation) + // If this is true, the activity should implement doInitialSoundAndAnimation + // if anything should autoplay when the page appears. + soundManagement?: boolean; } // This is the object (implemented by us, not the activity) that represents our own @@ -62,19 +69,28 @@ export class ActivityManager { this.builtInActivities[ simpleCheckboxQuizModule.dataActivityID ] = simpleCheckboxQuizModule as IActivityModule; - this.builtInActivities[ - "drag-to-destination" - ] = dragToDestinationModule as IActivityModule; - // Review: currently these two use the same module. A lot of stuff is shared, all the way down to the + + // Review: currently these all use the same module. A lot of stuff is shared, all the way down to the // prepareActivity() function in dragActivityRuntime. But a good many specialized TOP types are - // specific to one of the three and not needed for the others. It may be helpful to tease things + // specific to one of them and not needed for the others. It may be helpful to tease things // apart more, for example, three separate implementations of IActivityModule and PrepareActivity - // which call common code for the setup tasks common to all three. + // which call common code for the setup tasks common to all three. OTOH, in some ways it is simpler + // to have it all in one place, and just do the appropriate initialization based on what kind of + // draggables we find. + this.builtInActivities[ + "drag-to-destination" // not currently used + ] = dragToDestinationModule as IActivityModule; + this.builtInActivities[ + "drag-letter-to-target" + ] = dragToDestinationModule as IActivityModule; + this.builtInActivities[ + "drag-image-to-target" + ] = dragToDestinationModule as IActivityModule; this.builtInActivities[ - "sort-sentence" + "drag-sort-sentence" ] = dragToDestinationModule as IActivityModule; this.builtInActivities[ - "word-chooser-slider" + "word-chooser-slider" // not used yet ] = dragToDestinationModule as IActivityModule; } public getActivityAbsorbsDragging(): boolean { @@ -221,6 +237,14 @@ export class ActivityManager { } } + public doInitialSoundAndAnimation() { + if (this.currentActivity && this.currentActivity.runningObject) { + this.currentActivity.runningObject.doInitialSoundAndAnimation?.( + this.currentActivity.context! + ); + } + } + // Showing a new page, so stop any previous activity and start any new one that might be on the new page. // returns true if this is a page where we are going to have state in the DOM so that the // container needs to be careful not to get rid of it to save memory. diff --git a/src/activities/dragActivities/DragToDestination.ts b/src/activities/dragActivities/DragToDestination.ts index fee893fa..22bd0e14 100644 --- a/src/activities/dragActivities/DragToDestination.ts +++ b/src/activities/dragActivities/DragToDestination.ts @@ -1,6 +1,7 @@ import { ActivityContext } from "../ActivityContext"; import { IActivityObject, IActivityRequirements } from "../activityManager"; import { + playInitialElements, prepareActivity, undoPrepareActivity } from "../../dragActivityRuntime"; @@ -9,9 +10,10 @@ import { const activityCss = require("!!raw-loader!./multipleChoiceDomActivity.css") .default;*/ -// This class is intentionally very generic. All it needs is that the html of the -// page it is given should have some objects (typically bloom-textOverPicture) that have -// data-correct-position. These objects are made draggable...[Todo: document more of this as we implement] +// This class is basically an adapter that implements IActivityObject so that activities that +// are created by Bloom's DragActivityTool (the 2024 Bloom Games) can connect to the +// dragActivityRuntime.ts functions that actually do the work of the activity (and are shared with +// Bloom desktop's Play mode).) // Note that you won't find any code using this directly. Instead, // it gets used by the ActivityManager as the default export of this module. @@ -23,7 +25,6 @@ export default class DragToDestinationActivity implements IActivityObject { // which is important because the user could be either // coming back to this page, or going to another instance of this activity // in a subsequent page. - // eslint-disable-next-line no-unused-vars public constructor(public pageElement: HTMLElement) {} public showingPage(activityContext: ActivityContext) { @@ -31,6 +32,10 @@ export default class DragToDestinationActivity implements IActivityObject { this.prepareToDisplayActivityEachTime(activityContext); } + public doInitialSoundAndAnimation(activityContext: ActivityContext) { + playInitialElements(activityContext.pageElement); + } + // Do just those things that we only want to do once per read of the book. // In the current implementation of activityManager, this is operating on a copy of the page html, // NOT the real DOM the user will eventually interact with. @@ -39,16 +44,19 @@ export default class DragToDestinationActivity implements IActivityObject { // The context removes event listeners each time the page is shown, so we have to put them back. private prepareToDisplayActivityEachTime(activityContext: ActivityContext) { this.activityContext = activityContext; - // These classes are added one layer outside the page body. This is an element that is a wrapper + // This class is added one layer outside the page body. This is an element that is a wrapper // for our scoped styles...the furthest out element we can put classes on and have them work // properly with scoped styles. It's a good place to put classes that affect the state of everything - // in the page. + // in the page. This class indicates in Bloom Desktop that the page is in play mode. + // In Bloom Player, it always is. This helps make a common stylesheet work consistently. + // Bloom Player may also add drag-activity-correct or drag-activity-wrong to this element, + // after checking an answer, or drag-activity-solution when showing the answer. activityContext.pageElement.parentElement?.classList.add( - "drag-activity-try-it", - "drag-activity-start" + "drag-activity-play" ); prepareActivity(activityContext.pageElement, next => { - // Move to the next or previous page. + // Move to the next or previous page. None of our current bloom game activities use this, but it's available + // if we create an activity with built-in next/previous page buttons. if (next) { activityContext.navigateToNextPage(); } else { @@ -63,10 +71,11 @@ export default class DragToDestinationActivity implements IActivityObject { if (this.activityContext) { undoPrepareActivity(this.activityContext.pageElement); this.activityContext.pageElement.parentElement?.classList.remove( - "drag-activity-try-it", - "drag-activity-start", + "drag-activity-play", + "drag-activity-start", // I don't think Bloom Player will ever add this, but just in case. "drag-activity-correct", - "drag-activity-wrong" + "drag-activity-wrong", + "drag-activity-solution" ); } } diff --git a/src/bloom-player-core.tsx b/src/bloom-player-core.tsx index e18f2d16..f97be442 100644 --- a/src/bloom-player-core.tsx +++ b/src/bloom-player-core.tsx @@ -51,8 +51,8 @@ import { kLocalStorageBookUrlKey } from "./bloomPlayerAnalytics"; import { autoPlayType } from "./bloom-player-controls"; -import { setCurrentPage } from "./narration"; import { + setCurrentPage as setCurrentNarrationPage, currentPlaybackMode, setCurrentPlaybackMode, PlaybackMode, @@ -65,10 +65,10 @@ import { PlayFailed, PlayCompleted, ToggleImageDescription, - pause, - getCurrentPage, - play, - hidingPage, + pauseNarration, + getCurrentNarrationPage, + playNarration, + hidingPage as hidingNarrationPage, pageHasAudio, setIncludeImageDescriptions, playAllSentences @@ -473,17 +473,18 @@ export class BloomPlayerCore extends React.Component { ? this.sourceUrl : this.sourceUrl + "/" + filename + ".htm"; - let urlPrefixT = haveFullPath + this.urlPrefix = haveFullPath ? this.sourceUrl.substring( 0, Math.max(slashIndex, encodedSlashIndex) ) : this.sourceUrl; - if (!urlPrefixT.startsWith("http")) { + if (!this.urlPrefix.startsWith("http")) { // Only in storybook with local books? - urlPrefixT = window.location.origin + "/" + urlPrefixT; + this.urlPrefix = + window.location.origin + "/" + this.urlPrefix; } - this.music.urlPrefix = this.urlPrefix = urlPrefixT; + this.music.urlPrefix = this.urlPrefix; setPlayerUrlPrefix(this.music.urlPrefix); // Note: this does not currently seem to work when using the storybook fileserver. // I hypothesize that it automatically filters files starting with a period, @@ -544,10 +545,6 @@ export class BloomPlayerCore extends React.Component { } this.animation.PlayAnimations = this.bookInfo.playAnimations; - console.log( - "animation.PlayAnimations", - this.animation.PlayAnimations - ); this.collectBodyAttributes(body); this.makeNonEditable(body); @@ -1145,13 +1142,9 @@ export class BloomPlayerCore extends React.Component { this.handleToggleImageDescription.bind(this) ); // allows narration to ask whether swiping to this page is still in progress. + // This doesn't seem to be super reliable, so that narration code also keeps track of + // how long it's been since we switched pages. setTestIsSwipeInProgress(() => { - console.log( - "animating: " + - this.swiperInstance + - " " + - this.swiperInstance?.animating - ); return this.swiperInstance?.animating; }); setLogNarration(url => logSound(url, 1)); @@ -1202,7 +1195,7 @@ export class BloomPlayerCore extends React.Component { // This test determines if we changed pages while paused, // since the narration object won't yet be updated. if ( - BloomPlayerCore.currentPage !== getCurrentPage() || + BloomPlayerCore.currentPage !== getCurrentNarrationPage() || currentPlaybackMode === PlaybackMode.MediaFinished ) { this.resetForNewPageAndPlay(BloomPlayerCore.currentPage!); @@ -1210,7 +1203,7 @@ export class BloomPlayerCore extends React.Component { if (currentPlaybackMode === PlaybackMode.VideoPaused) { this.video.play(); // sets currentPlaybackMode = VideoPlaying } else { - play(); // sets currentPlaybackMode = AudioPlaying + playNarration(); // sets currentPlaybackMode = AudioPlaying this.animation.PlayAnimation(); this.music.play(); } @@ -1437,11 +1430,10 @@ export class BloomPlayerCore extends React.Component { } private pauseAllMultimedia() { - const temp = currentPlaybackMode; if (currentPlaybackMode === PlaybackMode.VideoPlaying) { this.video.pause(); // sets currentPlaybackMode = VideoPaused } else if (currentPlaybackMode === PlaybackMode.AudioPlaying) { - pause(); // sets currentPlaybackMode = AudioPaused + pauseNarration(); // sets currentPlaybackMode = AudioPaused this.animation.PauseAnimation(); } // Music keeps playing after all video, narration, and animation have finished. @@ -2400,7 +2392,7 @@ export class BloomPlayerCore extends React.Component { // its continued playing. this.video.hidingPage(); this.video.HandlePageBeforeVisible(bloomPage); - hidingPage(); + hidingNarrationPage(); this.music.hidingPage(); if ( currentPlaybackMode === PlaybackMode.AudioPaused || @@ -2798,7 +2790,7 @@ export class BloomPlayerCore extends React.Component { } } this.animation.PlayAnimation(); // get rid of classes that made it pause - setCurrentPage(bloomPage); + setCurrentNarrationPage(bloomPage); // State must be set before calling HandlePageVisible() and related methods. if (BloomPlayerCore.currentPageHasVideo) { setCurrentPlaybackMode(PlaybackMode.VideoPlaying); @@ -2813,6 +2805,7 @@ export class BloomPlayerCore extends React.Component { public playAudioAndAnimation(bloomPage: HTMLElement | undefined) { if (this.activityManager.getActivityManagesSound()) { + this.activityManager.doInitialSoundAndAnimation(); return; // we don't just want to play them all, the activity code will do it selectively. } setCurrentPlaybackMode(PlaybackMode.AudioPlaying); diff --git a/src/dragActivityRuntime.ts b/src/dragActivityRuntime.ts index d02fd5a1..5fadf27e 100644 --- a/src/dragActivityRuntime.ts +++ b/src/dragActivityRuntime.ts @@ -61,6 +61,10 @@ export function prepareActivity( ) { currentPage = page; currentChangePageAction = changePageAction; + doShowAnswersInTargets( + page.getAttribute("data-show-answers-in-targets") === "true", + page + ); // not sure we need this in BP, but definitely for when Bloom desktop goes to another tab. savePositions(page); @@ -143,6 +147,11 @@ export function prepareActivity( elt.addEventListener("click", showCorrect); }); + const soundItems = Array.from(page.querySelectorAll("[data-sound]")); + soundItems.forEach((elt: HTMLElement) => { + elt.addEventListener("click", playSoundOf); + }); + prepareOrderSentenceActivity(page); // Slider: // for drag-word-chooser-slider @@ -156,8 +165,6 @@ export function prepareActivity( // showARandomWord(page, false); // setupSliderImageEvents(page); - - playInitialElements(page); } // Break any order-sentence element into words and @@ -229,17 +236,31 @@ export function undoPrepareActivity(page: HTMLElement) { elt.removeEventListener("click", performTryAgain); }); + const soundItems = Array.from(page.querySelectorAll("[data-sound]")); + soundItems.forEach((elt: HTMLElement) => { + elt.removeEventListener("click", playSoundOf); + }); + Array.from( page.getElementsByClassName("drag-item-random-sentence") ).forEach((elt: HTMLElement) => { elt.parentElement?.removeChild(elt); }); + doShowAnswersInTargets(true, page); //Slider: setSlideablesVisibility(page, true); // Array.from(page.getElementsByTagName("img")).forEach((img: HTMLElement) => { // img.removeEventListener("click", clickSliderImage); // }); } +const playSoundOf = (e: MouseEvent) => { + const elt = e.currentTarget as HTMLElement; + const soundFile = elt.getAttribute("data-sound"); + if (soundFile) { + playSound(elt, soundFile); + } +}; + const playAudioOfTarget = (e: PointerEvent) => { const target = e.currentTarget as HTMLElement; playAudioOf(target); @@ -248,11 +269,7 @@ const playAudioOfTarget = (e: PointerEvent) => { const playAudioOf = (element: HTMLElement) => { const possibleElements = getVisibleEditables(element); const playables = getAudioSentences(possibleElements); - playAllAudio(playables, getPage(element)); -}; - -const getPage = (element: HTMLElement): HTMLElement => { - return element.closest(".bloom-page") as HTMLElement; + playAllAudio(playables, element.closest(".bloom-page") as HTMLElement); }; function makeWordItems( @@ -289,7 +306,7 @@ function changePageButtonClicked(e: MouseEvent) { currentChangePageAction?.(next); } -function playInitialElements(page: HTMLElement) { +export function playInitialElements(page: HTMLElement) { const initialFilter = e => { const top = e.closest(".bloom-textOverPicture") as HTMLElement; if (!top) { @@ -589,29 +606,32 @@ function showCorrectOrWrongItems(page: HTMLElement, correct: boolean) { playAllVideo(videoElements, () => playAllAudio(playables, page)); }; if (soundFile) { - const audio = new Audio(urlPrefix() + "/audio/" + soundFile); - audio.style.visibility = "hidden"; - // To my surprise, in BP storybook it works without adding the audio to any document. - // But in Bloom proper, it does not. I think it is because this code is part of the toolbox, - // so the audio element doesn't have the right context to interpret the relative URL. - page.append(audio); - // It feels cleaner if we remove it when done. This could fail, e.g., if the user - // switches tabs or pages before we get done playing. Removing it immediately - // prevents the sound being played. It's not a big deal if it doesn't get removed. - audio.play(); - audio.addEventListener( - "ended", - () => { - page.removeChild(audio); - playOtherStuff(); - }, - { once: true } - ); + playSound(page, soundFile); } else { playOtherStuff(); } } +function playSound(someElt: HTMLElement, soundFile: string) { + const audio = new Audio(urlPrefix() + "/audio/" + soundFile); + audio.style.visibility = "hidden"; + // To my surprise, in BP storybook it works without adding the audio to any document. + // But in Bloom proper, it does not. I think it is because this code is part of the toolbox, + // so the audio element doesn't have the right context to interpret the relative URL. + someElt.append(audio); + audio.play(); + // It feels cleaner if we remove it when done. This could fail, e.g., if the user + // switches tabs or pages before we get done playing. Removing it immediately + // prevents the sound being played. It's not a big deal if it doesn't get removed. + audio.addEventListener( + "ended", + () => { + someElt.removeChild(audio); + }, + { once: true } + ); +} + function checkDraggables(page: HTMLElement) { let allCorrect = true; const draggables = Array.from(page.querySelectorAll("[data-bubble-id]")); @@ -900,6 +920,81 @@ function checkRandomSentences(page: HTMLElement) { return true; } +export const doShowAnswersInTargets = (showNow: boolean, page: HTMLElement) => { + const draggables = Array.from(page.querySelectorAll("[data-bubble-id]")); + if (showNow) { + draggables.forEach(draggable => { + copyContentToTarget(draggable as HTMLElement); + }); + } else { + draggables.forEach(draggable => { + removeContentFromTarget(draggable as HTMLElement); + }); + } +}; + +export function copyContentToTarget(draggable: HTMLElement) { + const target = getTarget(draggable); + if (!target) { + return; + } + // We want to copy the content of the draggale, with several exceptions. + // To reduce flicker, we do the manipulations on a temporary element, and + // only copy into the actual target if there is actually a change. + // (Flicker is particularly likely with changes that don't affect the + // target, like adding and removing the image editing buttons.) + const temp = target.ownerDocument.createElement("div"); + temp.innerHTML = draggable.innerHTML; + + // Don't need the bubble controls + Array.from(temp.getElementsByClassName("bloom-ui")).forEach(e => { + e.remove(); + }); + // Nor the image editing controls. + Array.from(temp.getElementsByClassName("imageOverlayButton")).forEach(e => { + e.remove(); + }); + Array.from(temp.getElementsByClassName("imageButton")).forEach(e => { + e.remove(); + }); + // Bloom has integrity checks for duplicate ids, and we don't need them in the duplicate content. + Array.from(temp.querySelectorAll("[id]")).forEach(e => { + e.removeAttribute("id"); + }); + Array.from(temp.getElementsByClassName("hoverUp")).forEach(e => { + // Produces at least a change in background color that we don't want. + e.classList.remove("hoverUp"); + }); + // Content is not editable inside the target. + Array.from(temp.querySelectorAll("[contenteditable]")).forEach(e => { + e.removeAttribute("contenteditable"); + }); + // Nor should we able to tab to it, or focus it. + Array.from(temp.querySelectorAll("[tabindex]")).forEach(e => { + e.removeAttribute("tabindex"); + }); + if (target.innerHTML !== temp.innerHTML) { + target.innerHTML = temp.innerHTML; + } +} + +export const getTarget = (draggable: HTMLElement): HTMLElement | undefined => { + const targetId = draggable.getAttribute("data-bubble-id"); + if (!targetId) { + return undefined; + } + return draggable.ownerDocument.querySelector( + `[data-target-of="${targetId}"]` + ) as HTMLElement; +}; + +function removeContentFromTarget(draggable: HTMLElement) { + const target = getTarget(draggable); + if (target) { + target.innerHTML = ""; + } +} + export let draggingSlider = false; // Setup that is common to Play and design time diff --git a/src/event.ts b/src/event.ts index 2cfea562..e428bfa1 100644 --- a/src/event.ts +++ b/src/event.ts @@ -1,4 +1,6 @@ -// originally from http://stackoverflow.com/a/14657922/723299 +// originally from http://stackoverflow.com/a/14657922/723299. +// Temporarily duplicated in Bloom Player and Bloom Desktop (under Drag Activity), +// but planned to become a Bloom Player artifact. interface ILiteEvent { subscribe(handler: (data?: T) => void): void; diff --git a/src/narration.ts b/src/narration.ts index 677ffd83..03325d25 100644 --- a/src/narration.ts +++ b/src/narration.ts @@ -1,6 +1,7 @@ // This file contains code for playing audio in a bloom page, including a draggable page. // The file is designed to be shared between Bloom Desktop and Bloom Player. Maybe eventually // as an npm module, but for now, just copied into both projects. See comments in dragActivityRuntime.ts. +// Until that day, changes must be copied manually, and care taken to avoid conflicts. // It is quite difficult to know how to handle audio in a drag activity page. // We need to be able to play it both during "Play" and when showing the page in BP. @@ -189,7 +190,8 @@ export function setCurrentPage(page: HTMLElement) { }, 3000); currentPlayPage = page; } -export function getCurrentPage(): HTMLElement { +// Get the page that the narration system thinks is current. +export function getCurrentNarrationPage(): HTMLElement { return currentPlayPage!; } @@ -205,7 +207,10 @@ let startPause: Date; // Gets canceled if we pause and restarted if we resume. let fakeNarrationTimer: number; -let segments: HTMLElement[]; +// List of segments that are currently being played, or will be resumed (or restarted) when play resumes. +// Typically the output of getPageAudioElements, but Bloom Games sometimes gives a different list. +// Unlike elementsToPlayConsecutivelyStack, this list is not reversed, nor do we remove items from it as we play them. +let segmentsWeArePlaying: HTMLElement[]; let currentAudioId = ""; @@ -256,15 +261,19 @@ export function playAllSentences(page: HTMLElement | null): void { export function playAllAudio(elements: HTMLElement[], page: HTMLElement): void { setCurrentPage(page); - segments = getPageAudioElements(page); + segmentsWeArePlaying = elements; startPlay = new Date(); const mediaPlayer = getPlayer(); if (mediaPlayer) { - // This felt like a good idea. But we are about to set a new src on the media player and play that, + // This felt like a good idea to do always. But we are about to set a new src on the media player and play that, // which will deal with any sound that is still playing. // And if we explicitly pause it now, that actually starts an async process of getting it paused, which // may not have completed by the time we attempt to play the new audio. And then play() fails. - //mediaPlayer.pause(); + // OTOH, if there is nothing new to play, we should terminate anything that is playing + // (perhaps from a previous page). + if (elements.length == 0) { + mediaPlayer.pause(); + } mediaPlayer.currentTime = 0; } @@ -299,12 +308,19 @@ export function playAllAudio(elements: HTMLElement[], page: HTMLElement): void { } const firstElementToPlay = elementsToPlayConsecutivelyStack[stackSize - 1]; // Remember to pop it when you're done playing it. (i.e., in playEnded) - // I didn't comment this at the time, but my recollection is that making a new player - // each time helps with some cases where the old one was in a bad state, - // such as in the middle of pausing. Don't do this between setting highlight and playing, though, + // At one point it seemed to help something to delete the media player and make a new one each time. + // I didn't comment this at the time, but my recollection is that this could help with some cases + // where the old one was in a bad state, such as in the middle of pausing. + // Currently, though, we're being more careful not to pause except when there is nothing more to play currently, + // (or when the user clicks the button), + // since changing the src will stop any old play, but pausing right before setting a new src and calling play() + // can cause the play() to fail. And somehow, deleting the media player here before we set up for a new play + // was causing play to fail, reporting an abort because the media was removed from the document. + // I don't fully understand why that was happening, but for now, things seem to be working best by just + // continuing to use the same player as long as it can be found. + // For sure, don't delete the player and make a new one between setting highlight and playing, // or the handler that removes the highlight suppression will be lost. - const src = mediaPlayer.getAttribute("src") ?? ""; - mediaPlayer.remove(); + //mediaPlayer.remove(); setSoundAndHighlight(firstElementToPlay, true); // Review: do we need to do something to let the rest of the world know about this? @@ -485,7 +501,7 @@ function playCurrentInternal() { if (currentPlaybackMode === PlaybackMode.AudioPlaying) { const mediaPlayer = getPlayer(); if (mediaPlayer) { - const element = getCurrentPage().querySelector( + const element = getCurrentNarrationPage().querySelector( `#${currentAudioId}` ); if (!element || !canPlayAudio(element)) { @@ -713,12 +729,12 @@ function onSubElementHighlightTimeEnded(originalSessionNum: number) { // Removes the .ui-audioCurrent class from all elements (also ui-audioCurrentImg) // Equivalent of removeAudioCurrentFromPageDocBody() in BloomDesktop. +// "around" might be the element that has the highlight, or the one getting it; +// the important thing is that it belongs to the right document (which is in question +// with multiple iframes in Bloom desktop). function removeAudioCurrent(around: HTMLElement = document.body) { // Note that HTMLCollectionOf's length can change if you change the number of elements matching the selector. - // For safety we get rid of all existing ones. But we do take a starting point element - // (might be the one that has the higlight, or the one getting it) - // to make sure we're cleaning up in the right document, which is in question when used in - // Bloom Editor. + // For safety we get rid of all existing ones. const audioCurrentArray = Array.from( around.ownerDocument.getElementsByClassName("ui-audioCurrent") ); @@ -726,7 +742,9 @@ function removeAudioCurrent(around: HTMLElement = document.body) { for (let i = 0; i < audioCurrentArray.length; i++) { audioCurrentArray[i].classList.remove("ui-audioCurrent"); } - const currentImg = document.getElementsByClassName("ui-audioCurrentImg")[0]; + const currentImg = around.ownerDocument.getElementsByClassName( + "ui-audioCurrentImg" + )[0]; if (currentImg) { currentImg.classList.remove("ui-audioCurrentImg"); } @@ -1104,14 +1122,20 @@ export function pageHasAudio(page: HTMLElement): boolean { return getPageAudioElements(page).length ? true : false; } -export function play() { +// Called when the user clicks the play/pause button, and we want to resume playing. +// If we're in the middle of playing, we resume it. +// If we have finished playing, we start over. +// If the page nas no audio, we assume the user paused as long as wanted on +// the page, and raise the PageNarrationComplete event at once (to move to the +// next page if we are in autoplay). +export function playNarration() { if (currentPlaybackMode === PlaybackMode.AudioPlaying) { return; // no change. } setCurrentPlaybackMode(PlaybackMode.AudioPlaying); // I'm not sure how getPlayer() can return null/undefined, but have seen it happen // typically when doing something odd like trying to go back from the first page. - if (segments.length && getPlayer()) { + if (segmentsWeArePlaying.length && getPlayer()) { if (elementsToPlayConsecutivelyStack.length) { handlePlayPromise(getPlayer().play()); @@ -1128,27 +1152,12 @@ export function play() { } } // Nothing real to play on this page, so PageNarrationComplete depends on a timeout. - // adjust startPlay by the elapsed pause. From this compute how much of durationOfPagesWithoutNarration - // remains. (I don't think this is quite right. It seems to always jump straight to the next page - // when resumed. But at least it is possible to pause on a page with no narration, which is better than - // the previous version.) - const pause = new Date().getTime() - startPause.getTime(); - startPlay = new Date(startPlay.getTime() + pause); - const remaining = - durationOfPagesWithoutNarration - - (new Date().getTime() - startPlay.getTime()); - if (remaining > 0) { - fakeNarrationTimer = window.setTimeout(() => { - setCurrentPlaybackMode(PlaybackMode.MediaFinished); - PageNarrationComplete?.raise(currentPlayPage!); - }, remaining); - } else { - // Somehow we already reached the limit. - PageNarrationComplete?.raise(currentPlayPage!); - } + // We only get here following a pause, so assume the reader has paused as long as wanted, + // and move on. + PageNarrationComplete?.raise(currentPlayPage!); } -export function pause() { +export function pauseNarration() { if (currentPlaybackMode === PlaybackMode.AudioPaused) { return; } @@ -1170,7 +1179,7 @@ function pausePlaying() { const player = getPlayer(); // We're paused, so if we have a timer running to switch pages after a certain time, cancel it. clearTimeout(fakeNarrationTimer); - if (segments && segments.length && player) { + if (segmentsWeArePlaying && segmentsWeArePlaying.length && player) { // Before reporting duration, try to check that we really are playing. // a separate report is sent if play ends. if (player.currentTime > 0 && !player.paused && !player.ended) { @@ -1202,25 +1211,30 @@ export function computeDuration(page: HTMLElement): number { } export function hidingPage() { - pausePlaying(); // Doesn't set AudioPaused state. Caller sets NewPage state. + // This causes problems. When we're hiding one page, we immediately show another. + // If that page has no audio, we pause the player then. + // If it DOES have audio, a pause here can interfere with playing it. + //pausePlaying(); // Doesn't set AudioPaused state. Caller sets NewPage state. clearTimeout(fakeNarrationTimer); } // Play the specified elements, one after the other. When the last completes (or at once if the array is empty), // perform the 'then' action (typically used to play narration, which we put after videos). // Todo: Bloom Player version, at least, should work with play/pause/resume/change page architecture. +// (This function would be more natural in video.ts. But at least for now I'm trying to minimize the +// number of source files shared with Bloom Desktop, and we need this for Bloom Games.) export function playAllVideo(elements: HTMLVideoElement[], then: () => void) { if (elements.length === 0) { then(); return; } const video = elements[0]; - // Note: sometimes this event does not fire normally, even when the video is played to the end. + // Note: in Bloom Desktop, sometimes this event does not fire normally, even when the video is played to the end. // I have not figured out why. It may be something to do with how we are trimming them. - // In Bloom, this is worked around by raising the ended event when we detect that it has paused past the end point + // In Bloom Desktop, this is worked around by raising the ended event when we detect that it has paused past the end point // in resetToStartAfterPlayingToEndPoint. - // In BloomPlayer, we may need to do something similar. Or possibly it's not a problem because export - // really shortens videos rather than just having BP limit the playback time. + // In BloomPlayer,I don't think this is a problem. Videos are trimmed when published, so we always play to the + // real end (unless the user pauses). So one way or another, we should get the ended event. video.addEventListener( "ended", () => { diff --git a/src/narrationUtils.ts b/src/narrationUtils.ts deleted file mode 100644 index e2b4de00..00000000 --- a/src/narrationUtils.ts +++ /dev/null @@ -1,1185 +0,0 @@ -import LiteEvent from "./event"; - -export interface ISetHighlightParams { - newElement: Element; - shouldScrollToElement: boolean; - disableHighlightIfNoAudio?: boolean; - oldElement?: Element | null | undefined; // Optional. Provides some minor optimization if set. -} - -export enum PlaybackMode { - NewPage, // starting a new page ready to play - NewPageMediaPaused, // starting a new page in the "paused" state - VideoPlaying, // video is playing - VideoPaused, // video is paused - AudioPlaying, // narration and/or animation are playing (or possibly finished) - AudioPaused, // narration and/or animation are paused - MediaFinished // video, narration, and/or animation has played (possibly no media to play) - // Note that music can be playing when the state is either AudioPlaying or MediaFinished. -} - -export let currentPlaybackMode: PlaybackMode; -export function setCurrentPlaybackMode(mode: PlaybackMode) { - currentPlaybackMode = mode; -} - -// These functions support allowing a client (typically BloomPlayerCore) to register as -// the object that wants to receive notification of how long audio was played. -// Duration is in seconds. -export let durationReporter: (duration: number) => void; -export function listenForPlayDuration(reporter: (duration: number) => void) { - durationReporter = reporter; -} - -// A client may configure a function which can be called to find out whether a swipe -// is in progress...in BloomPlayerCore, this is implemented by a test on SWiper. -// It is currently only used when we need to scroll the content of a field we are -// playing. Bloom Desktop does not need to set it. -export let isSwipeInProgress: () => boolean; -export function setTestIsSwipeInProgress(tester: () => boolean) { - isSwipeInProgress = tester; -} - -// A client may configure a function which is passed the URL of each audio file narration plays. -export let logNarration: (src: string) => void; -export function setLogNarration(logger: (src: string) => void) { - logNarration = logger; -} - -let playerUrlPrefix = ""; -// In bloom player, figuring the url prefix is more complicated. We pass it in. -// In Bloom desktop, we don't call this at all. The code that would naturally do it -// is in the wrong iframe and it's a pain to get it to the right one. -// But there, the urlPrevix function works fine. -export function setPlayerUrlPrefix(prefix: string) { - playerUrlPrefix = prefix; -} - -export function urlPrefix(): string { - if (playerUrlPrefix) { - return playerUrlPrefix; - } - const bookSrc = window.location.href; - const index = bookSrc.lastIndexOf("/"); - const bookFolderUrl = bookSrc.substring(0, index); - return bookFolderUrl; -} - -// We need to sort these by the tabindex of the containing bloom-translationGroup element. -// We need a stable sort, which array.sort() does not provide: elements in the same -// bloom-translationGroup, or where the translationGroup does not have a tabindex, -// should remain in their current order. -// This function was extracted from the narration.ts file for testing: -// I was not able to import the narration file into the test suite because -// MutationObserver is not emulated in the test environment. -// It's not obvious what should happen to TGs with no tabindex when others have it. -// At this point we're going with the approach that no tabindex is equivalent to tabindex 999. -// This should cause text with no tabindex to sort to the bottom, if other text has a tabindex; -// It should also not affect order in situations where no text has a tabindex -// (An earlier algorithm attempted to preserve document order for the no-tab-index case -// by comparing any two elements using document order if either lacks tabindex. -// This works well for many cases, but if there's a no-tabindex element between two -// that get re-ordered (e.g., ABCDEF where the only tabindexes are C=2 and E=1), -// the function is not transitive (e.g. C < D < E but E < C) which will produce -// unpredictable results. -export function sortAudioElements(input: HTMLElement[]): HTMLElement[] { - const keyedItems = input.map((item, index) => { - return { tabindex: getTgTabIndex(item), index, item }; - }); - keyedItems.sort((x, y) => { - // If either is not in a translation group with a tabindex, - // order is determined by their original index. - // Likewise if the tabindexes are the same. - if (!x.tabindex || !y.tabindex || x.tabindex === y.tabindex) { - return x.index - y.index; - } - // Otherwise, determined by the numerical order of tab indexes. - return parseInt(x.tabindex, 10) - parseInt(y.tabindex, 10); - }); - return keyedItems.map(x => x.item); -} - -function getTgTabIndex(input: HTMLElement): string | null { - let tg: HTMLElement | null = input; - while (tg && !tg.classList.contains("bloom-translationGroup")) { - tg = tg.parentElement; - } - if (!tg) { - return "999"; - } - return tg.getAttribute("tabindex") || "999"; -} - -// ------migrated from dragActivityNarration, common in narration.ts - -// Even though these can now encompass more than strict sentences, -// we continue to use this class name for backwards compatability reasons. -export const kAudioSentence = "audio-sentence"; -const kSegmentClass = "bloom-highlightSegment"; -const kImageDescriptionClass = "bloom-imageDescription"; -// Indicates that highlighting is briefly/temporarily suppressed, -// but may become highlighted later. -// For example, audio highlighting is suppressed until the related audio starts playing (to avoid flashes) -const kSuppressHighlightClass = "ui-suppressHighlight"; -// This event allows Narration to inform its controllers when we start/stop reading -// image descriptions. It is raised for each segment we read and passed true if the one -// we are about to read is an image description, false otherwise. -// Todo: wants a better name, it's not about toggling whether something is an image description, -// but about possibly updating the UI to reflect whether we are reading one. -export const ToggleImageDescription = new LiteEvent(); -const PageDurationAvailable = new LiteEvent; - -// On a typical page with narration, these are raised at the same time, when the last narration -// on the page finishes. But if there is no narration at all, PlayCompleted will be raised -// immediately (useful for example to disable a pause button), but PageNarrationComplete will -// be raised only after the standard delay for non-audio page (useful for auto-advancing to the next page). -export const PageNarrationComplete = new LiteEvent; -export const PlayCompleted = new LiteEvent; -// Raised when we can't play narration, specifically because the browser won't allow it until -// the user has interacted with the page. -export const PlayFailed = new LiteEvent; -let currentAudioId = ""; - -// The first one to play should be at the end for both of these -let elementsToPlayConsecutivelyStack: HTMLElement[] = []; -let subElementsWithTimings: Array<[Element, number]> = []; - -let startPause: Date; -let segments: HTMLElement[]; - -// A Session Number that keeps track of each time playAllAudio started. -// This might be needed to keep track of changing pages, or when we start new audio -// that will replace something already playing. -let currentAudioSessionNum: number = 0; - -// This represents the start time of the current playing of the audio. If the user presses pause/play, it will be reset. -// This is used for analytics reporting purposes -let audioPlayCurrentStartTime: number | null = null; // milliseconds (since 1970/01/01, from new Date().getTime()) - -let currentPlayPage: HTMLElement | null = null; -// Unused in Bloom desktop, but in Bloom player, current page might change while a series of sounds -// is playing. This lets us avoid starting the next sound if the page has changed in the meantime. -export function setCurrentPage(page: HTMLElement) { - currentPlayPage = page; -} -export function getCurrentPage(): HTMLElement { - return currentPlayPage!; -} - -function playCurrentInternal() { - if (currentPlaybackMode === PlaybackMode.AudioPlaying) { - let mediaPlayer = getPlayer(); - if (mediaPlayer) { - const element = getCurrentPage().querySelector( - `#${currentAudioId}` - ); - if (!element || !canPlayAudio(element)) { - playEnded(); - return; - } - - // I didn't comment this at the time, but my recollection is that making a new player - // each time helps with some cases where the old one was in a bad state, - // such as in the middle of pausing. - const src = mediaPlayer.getAttribute("src") ?? ""; - mediaPlayer.remove(); - mediaPlayer = getPlayer(); - mediaPlayer.setAttribute("src", src); - - // Regardless of whether we end up using timingsStr or not, - // we should reset this now in case the previous page used it and was still playing - // when the user flipped to the next page. - subElementsWithTimings = []; - - const timingsStr: string | null = element.getAttribute( - "data-audioRecordingEndTimes" - ); - if (timingsStr) { - const childSpanElements = element.querySelectorAll( - `span.${kSegmentClass}` - ); - const fields = timingsStr.split(" "); - const subElementCount = Math.min( - fields.length, - childSpanElements.length - ); - - for (let i = subElementCount - 1; i >= 0; --i) { - const durationSecs: number = Number(fields[i]); - if (isNaN(durationSecs)) { - continue; - } - subElementsWithTimings.push([ - childSpanElements.item(i), - durationSecs - ]); - } - } else { - // No timings string available. - // No need for us to do anything. The correct element is already highlighted by playAllSentences() (which needed to call setCurrent... anyway to set the audio player source). - // We'll just proceed along, start playing the audio, and playNextSubElement() will return immediately because there are no sub-elements in this case. - } - - const currentSegment = element as HTMLElement; - if (currentSegment) { - ToggleImageDescription.raise( - isImageDescriptionSegment(currentSegment) - ); - } - - gotErrorPlaying = false; - console.log("playing " + currentAudioId + "..." + mediaPlayer.src); - const promise = mediaPlayer.play(); - ++currentAudioSessionNum; - audioPlayCurrentStartTime = new Date().getTime(); - highlightNextSubElement(currentAudioSessionNum); - handlePlayPromise(promise); - } - } -} - -function isImageDescriptionSegment(segment: HTMLElement): boolean { - return segment.closest("." + kImageDescriptionClass) !== null; -} - -function canPlayAudio(current: Element): boolean { - return true; // currently no way to check -} - -// Moves the highlight to the next sub-element -// originalSessionNum: The value of currentAudioSessionNum at the time when the audio file started playing. -// This is used to check in the future if the timeouts we started are for the right session. -// startTimeInSecs is an optional fallback that will be used in case the currentTime cannot be determined from the audio player element. -function highlightNextSubElement( - originalSessionNum: number, - startTimeInSecs: number = 0 -) { - // the item should not be popped off the stack until it's completely done with. - const subElementCount = subElementsWithTimings.length; - - if (subElementCount <= 0) { - return; - } - - const topTuple = subElementsWithTimings[subElementCount - 1]; - const element = topTuple[0]; - const endTimeInSecs: number = topTuple[1]; - - setHighlightTo({ - newElement: element, - shouldScrollToElement: true, - disableHighlightIfNoAudio: false - }); - - const mediaPlayer: HTMLMediaElement = document.getElementById( - "bloom-audio-player" - )! as HTMLMediaElement; - let currentTimeInSecs: number = mediaPlayer.currentTime; - if (currentTimeInSecs <= 0) { - currentTimeInSecs = startTimeInSecs; - } - - // Handle cases where the currentTime has already exceeded the nextStartTime - // (might happen if you're unlucky in the thread queue... or if in debugger, etc.) - // But instead of setting time to 0, set the minimum highlight time threshold to 0.1 (this threshold is arbitrary). - const durationInSecs = Math.max(endTimeInSecs - currentTimeInSecs, 0.1); - - setTimeout(() => { - onSubElementHighlightTimeEnded(originalSessionNum); - }, durationInSecs * 1000); -} - -function handlePlayPromise(promise: Promise, player?: HTMLMediaElement) { - // In newer browsers, play() returns a promise which fails - // if the browser disobeys the command to play, as some do - // if the user hasn't 'interacted' with the page in some - // way that makes the browser think they are OK with it - // playing audio. In Gecko45, the return value is undefined, - // so we mustn't call catch. - if (promise && promise.catch) { - promise.catch((reason: any) => { - // There is an error handler here, but the HTMLMediaElement also has an error handler (which may end up calling playEnded()). - // In case it doesn't, we make sure here that it happens - handlePlayError(); - // This promise.catch error handler is the only one that handles NotAllowedException (that is, playback not started because user has not interacted with the page yet). - // However, older versions of browsers don't support promise from HTMLMediaElement.play(). So this cannot be the only error handler. - // Thus we need both the promise.catch error handler as well as the HTMLMediaElement's error handler. - // - // In many cases (such as NotSupportedError, which happens when the audio file isn't found), both error handlers will run. - // That is a little annoying but if the two don't conflict with each other it's not problematic. - - const playingWhat = player?.getAttribute("src") ?? "unknown"; - console.log( - "could not play sound: " + reason + " " + playingWhat - ); - - if ( - reason && - reason - .toString() - .includes( - "The play() request was interrupted by a call to pause()." - ) - ) { - // We were getting this error Aug 2020. I tried wrapping the line above which calls mediaPlayer.play() - // (currently `promise = mediaPlayer.play();`) in a setTimeout with 0ms. This seemed to fix the bug (with - // landscape books not having audio play initially -- BL-8887). But the root cause was actually that - // we ended up calling playAllSentences twice when the book first loaded. - // I fixed that in bloom-player-core. But I wanted to document the possible setTimeout fix here - // in case this issue ever comes up for a different reason. - console.log( - "See comment in narration.ts for possibly useful information regarding this error." - ); - } - - // Don't call removeAudioCurrent() here. The HTMLMediaElement's error handler will call playEnded() and calling removeAudioCurrent() here will mess up playEnded(). - // removeAudioCurrent(); - - // With some kinds of invalid sound file it keeps trying and plays over and over. - // But when we move on to play another sound, a pause here will mess things up. - // So instead I put a pause after we run out of sounds to try to play. - //getPlayer().pause(); - // if (Pause) { - // Pause.raise(); - // } - - // Get all the state (and UI) set correctly again. - // Not entirely sure about limiting this to NotAllowedError, but that's - // the one kind of play error that is fixed by the user just interacting. - // If there's some other reason we can't play, showing as paused may not - // be useful. See comments on the similar code in music.ts - if (reason.name === "NotAllowedError") { - PlayFailed.raise(); - } - }); - } -} - -// Handles a timeout indicating that the expected time for highlighting the current subElement has ended. -// If we've really played to the end of that subElement, highlight the next one (if any). -// originalSessionNum: The value of currentAudioSessionNum at the time when the audio file started playing. -// This is used to check in the future if the timeouts we started are for the right session -function onSubElementHighlightTimeEnded(originalSessionNum: number) { - // Check if the user has changed pages since the original audio for this started playing. - // Note: Using the timestamp allows us to detect switching to the next page and then back to this page. - // Using playerPage (HTMLElement) does not detect that. - if (originalSessionNum !== currentAudioSessionNum) { - return; - } - // Seems to be needed to prevent jumping to the next subelement when not permitted to play by browser. - // Not sure why the check below on mediaPlayer.currentTime does not prevent this. - if (currentPlaybackMode === PlaybackMode.AudioPaused) { - return; - } - - const subElementCount = subElementsWithTimings.length; - if (subElementCount <= 0) { - return; - } - - const mediaPlayer: HTMLMediaElement = document.getElementById( - "bloom-audio-player" - )! as HTMLMediaElement; - if (mediaPlayer.ended || mediaPlayer.error) { - // audio playback ended. No need to highlight anything else. - // (No real need to remove the highlights either, because playEnded() is supposed to take care of that.) - return; - } - const playedDurationInSecs: number | undefined | null = - mediaPlayer.currentTime; - - // Peek at the next sentence and see if we're ready to start that one. (We might not be ready to play the next audio if the current audio got paused). - const subElementWithTiming = subElementsWithTimings[subElementCount - 1]; - const nextStartTimeInSecs = subElementWithTiming[1]; - - if (playedDurationInSecs && playedDurationInSecs < nextStartTimeInSecs) { - // Still need to wait. Exit this function early and re-check later. - const minRemainingDurationInSecs = - nextStartTimeInSecs - playedDurationInSecs; - setTimeout(() => { - onSubElementHighlightTimeEnded(originalSessionNum); - }, minRemainingDurationInSecs * 1000); - - return; - } - - subElementsWithTimings.pop(); - - highlightNextSubElement(originalSessionNum, nextStartTimeInSecs); -} - -function setSoundFrom(element: Element) { - const firstAudioSentence = getFirstAudioSentenceWithinElement(element); - const id: string = firstAudioSentence ? firstAudioSentence.id : element.id; - setCurrentAudioId(id); -} - -function setCurrentAudioId(id: string) { - if (!currentAudioId || currentAudioId !== id) { - currentAudioId = id; - updatePlayerStatus(); - } -} - -function updatePlayerStatus() { - const player = getPlayer(); - if (!player) { - return; - } - // Any time we change the src, the player will pause. - // So if we're playing currently, we'd better report whatever time - // we played. - if (player.currentTime > 0 && !player.paused && !player.ended) { - reportPlayDuration(); - } - const url = currentAudioUrl(currentAudioId); - logNarration(url); - player.setAttribute("src", url + "?nocache=" + new Date().getTime()); -} - -function getPlayer(): HTMLMediaElement { - const audio = getAudio("bloom-audio-player", _ => {}); - // We used to do this in the init call, but sometimes the function didn't get called. - // Suspecting that there are cases, maybe just in storybook, where a new instance - // of the narration object gets created, but the old audio element still exists. - // Make sure the current instance has our end function. - // Because it is a fixed function for the lifetime of this object, addEventListener - // will not add it repeatedly. - audio.addEventListener("ended", playEnded); - audio.addEventListener("error", handlePlayError); - // If we are suppressing hiliting something until we confirm that the audio really exists, - // we can stop doing so: the audio is playing. - audio.addEventListener("playing", removeHighlightSuppression); - return audio; -} -function removeHighlightSuppression() { - Array.from( - document.getElementsByClassName(kSuppressHighlightClass) - ).forEach(newElement => - newElement.classList.remove(kSuppressHighlightClass) - ); -} - -function currentAudioUrl(id: string): string { - const result = urlPrefix() + "/audio/" + id + ".mp3"; - console.log("trying to play " + result); - return result; -} - -function getAudio(id: string, init: (audio: HTMLAudioElement) => void) { - let player: HTMLAudioElement | null = document.querySelector( - "#" + id - ) as HTMLAudioElement; - if (player && !player.play) { - player.remove(); - player = null; - } - if (!player) { - player = document.createElement("audio") as HTMLAudioElement; - player.setAttribute("id", id); - document.body.appendChild(player); - init(player); - } - return player as HTMLMediaElement; -} - -function playEnded(): void { - // Not sure if this is necessary, since both 'playCurrentInternal()' and 'reportPlayEnded()' - // will toggle image description already, but if we've just gotten to the end of our "stack", - // it may be needed. - if (ToggleImageDescription) { - ToggleImageDescription.raise(false); - } - reportPlayDuration(); - if ( - elementsToPlayConsecutivelyStack && - elementsToPlayConsecutivelyStack.length > 0 - ) { - elementsToPlayConsecutivelyStack.pop(); // get rid of the last one we played - const newStackCount = elementsToPlayConsecutivelyStack.length; - if (newStackCount > 0) { - // More items to play - const nextElement = - elementsToPlayConsecutivelyStack[newStackCount - 1]; - setSoundAndHighlight(nextElement, true); - playCurrentInternal(); - } else { - reportPlayEnded(); - removeAudioCurrent(); - // In some error conditions, we need to stop repeating attempts to play. - getPlayer().pause(); - } - } -} - -function reportPlayEnded() { - elementsToPlayConsecutivelyStack = []; - subElementsWithTimings = []; - - removeAudioCurrent(); - PageNarrationComplete.raise(currentPlayPage!); - PlayCompleted.raise(); -} - -function reportPlayDuration() { - if (!audioPlayCurrentStartTime || !durationReporter) { - return; - } - const currentTime = new Date().getTime(); - const duration = (currentTime - audioPlayCurrentStartTime) / 1000; - durationReporter(duration); -} - -function setSoundAndHighlight( - newElement: Element, - disableHighlightIfNoAudio: boolean, - oldElement?: Element | null | undefined -) { - setHighlightTo({ - newElement, - shouldScrollToElement: true, // Always true in bloom-player version - disableHighlightIfNoAudio, - oldElement - }); - setSoundFrom(newElement); -} - -function setHighlightTo({ - newElement, - shouldScrollToElement, - disableHighlightIfNoAudio, - oldElement -}: ISetHighlightParams) { - // This should happen even if oldElement and newElement are the same. - if (shouldScrollToElement) { - // Wrap it in a try/catch so that if something breaks with this minor/nice-to-have feature of scrolling, - // the main responsibilities of this method can still proceed - try { - scrollElementIntoView(newElement); - } catch (e) { - console.error(e); - } - } - - if (oldElement === newElement) { - // No need to do much, and better not to, so that we can avoid any temporary flashes as the highlight is removed and re-applied - return; - } - - removeAudioCurrent(); - - if (disableHighlightIfNoAudio) { - const mediaPlayer = getPlayer(); - const isAlreadyPlaying = mediaPlayer.currentTime > 0; - - // If it's already playing, no need to disable (Especially in the Soft Split case, where only one file is playing but multiple sentences need to be highlighted). - if (!isAlreadyPlaying) { - // Start off in a highlight-disabled state so we don't display any momentary highlight for cases where there is no audio for this element. - // In react-based bloom-player, canPlayAudio() can't trivially identify whether or not audio exists, - // so we need to incorporate a derivative of Bloom Desktop's .ui-suppressHighlight code - newElement.classList.add(kSuppressHighlightClass); - } - } - - newElement.classList.add("ui-audioCurrent"); - // If the current audio is part of a (currently typically hidden) image description, - // highlight the image. - // it's important to check for imageDescription on the translationGroup; - // we don't want to highlight the image while, for example, playing a TOP box content. - const translationGroup = newElement.closest(".bloom-translationGroup"); - if ( - translationGroup && - translationGroup.classList.contains(kImageDescriptionClass) - ) { - const imgContainer = translationGroup.closest(".bloom-imageContainer"); - if (imgContainer) { - imgContainer.classList.add("ui-audioCurrentImg"); - } - } -} - -// Removes the .ui-audioCurrent class from all elements (also ui-audioCurrentImg) -// Equivalent of removeAudioCurrentFromPageDocBody() in BloomDesktop. -function removeAudioCurrent() { - // Note that HTMLCollectionOf's length can change if you change the number of elements matching the selector. - const audioCurrentCollection: HTMLCollectionOf = document.getElementsByClassName( - "ui-audioCurrent" - ); - - // Convert to an array whose length won't be changed - const audioCurrentArray: Element[] = Array.from(audioCurrentCollection); - - for (let i = 0; i < audioCurrentArray.length; i++) { - audioCurrentArray[i].classList.remove("ui-audioCurrent"); - } - const currentImg = document.getElementsByClassName("ui-audioCurrentImg")[0]; - if (currentImg) { - currentImg.classList.remove("ui-audioCurrentImg"); - } -} - -// Scrolls an element into view. -function scrollElementIntoView(element: Element) { - // In Bloom Player, scrollIntoView can interfere with page swipes, - // so Bloom Player needs some smarts about when to call it... - if (isSwipeInProgress?.()) { - // This alternative implementation doesn't use scrollIntoView (Which interferes with swiper). - // Since swiping is only active at the beginning (usually while the 1st element is playing) - // it should generally be good enough just to reset the scroll of the scroll parent to the top. - - // Assumption: Assumes the editable is the scrollbox. - // If this is not the case, you can use JQuery's scrollParent() function or other equivalent - const scrollAncestor = getEditable(element); - if (scrollAncestor) { - scrollAncestor.scrollTop = 0; - } - return; - } - - let mover = element as HTMLElement; // by default make the element itself scrollIntoView - if (window.getComputedStyle(element.parentElement!).position !== "static") { - // We can make a new element absolutely positioned and it will be relative to the parent. - // The idea is to make an element much narrower than the element we are - // trying to make visible, since we don't want horizontal movement. Quite possibly, - // as in BL-11038, only some white space is actually off-screen. But even if the author - // has positioned a bubble so some text is cut off, we don't want horizontal scrolling, - // which inside swiper will weirdly pull in part of the next page. - // (In the pathological case that the bubble is more than half hidden, we'll do the - // horizontal scroll, despite the ugliness of possibly showing part of the next page.) - // Note that elt may be a span, when scrolling chunks of text into view to play. - // I thought about using scrollWidth/Height to include any part of the element - // that is scrolled out of view, but for some reason these are always zero for spans. - // OffsetHeight seems to give the full height, though docs seem to indicate that it - // should not include invisible areas. - const elt = element as HTMLElement; - mover = document.createElement("div"); - mover.style.position = "absolute"; - mover.style.top = elt.offsetTop + "px"; - - // now we need what for a block would be offsetLeft. However, for a span, that - // yields the offset of the top left corner, which may be in the middle - // of a line. - const bounds = elt.getBoundingClientRect(); - const parent = elt.parentElement; - const parentBounds = parent?.getBoundingClientRect(); - const scale = parentBounds!.width / parent!.offsetWidth; - const leftRelativeToParent = (bounds.left - parentBounds!.left) / scale; - - mover.style.left = leftRelativeToParent + elt.offsetWidth / 2 + "px"; - mover.style.height = elt.offsetHeight + "px"; - mover.style.width = "0"; - element.parentElement?.insertBefore(mover, element); - } - - mover.scrollIntoView({ - // Animated instead of sudden - behavior: "smooth", - - // "nearest" setting does lots of smarts for us (compared to us deciding when to use "start" or "end") - // Seems to reduce unnecessary scrolling compared to start (aka true) or end (aka false). - // Refer to https://drafts.csswg.org/cssom-view/#scroll-an-element-into-view, - // which seems to imply that it won't do any scrolling if the two relevant edges are already inside. - block: "nearest" - - // horizontal alignment is controlled by "inline". We'll leave it as its default ("nearest") - // which typically won't move things at all horizontally - }); - if (mover !== element) { - mover.parentElement?.removeChild(mover); - } -} - -function getEditable(element: Element): Element | null { - if (element.classList.contains("bloom-editable")) { - return element; - } else { - return element.closest(".bloom-editable"); // Might be null - } -} - -// If something goes wrong playing a media element, typically that we don't actually have a recording -// for a particular one, we seem to sometimes get an error event, while other times, the promise returned -// by play() is rejected. Both cases call handlePlayError, which calls playEnded, but in case we get both, -// we don't want to call playEnded twice. -let gotErrorPlaying = false; - -function handlePlayError() { - if (gotErrorPlaying) { - console.log("Already got error playing, not handling again"); - return; - } - gotErrorPlaying = true; - console.log("Error playing, handling"); - setTimeout(() => { - playEnded(); - }, 100); -} - -function getFirstAudioSentenceWithinElement( - element: Element | null -): Element | null { - const audioSentences = getAudioSegmentsWithinElement(element); - if (!audioSentences || audioSentences.length === 0) { - return null; - } - - return audioSentences[0]; -} - -function getAudioSegmentsWithinElement(element: Element | null): Element[] { - const audioSegments: Element[] = []; - - if (element) { - if (element.classList.contains(kAudioSentence)) { - audioSegments.push(element); - } else { - const collection = element.getElementsByClassName(kAudioSentence); - for (let i = 0; i < collection.length; ++i) { - const audioSentenceElement = collection.item(i); - if (audioSentenceElement) { - audioSegments.push(audioSentenceElement); - } - } - } - } - - return audioSegments; -} - -// --------- migrated from narration.ts, not in dragActivityNarration - -let durationOfPagesWithoutNarration = 3.0; // seconds -export function setDurationOfPagesWithoutNarration(d: number) { - durationOfPagesWithoutNarration = d; -} -let includeImageDescriptions: boolean = true; -export function setIncludeImageDescriptions(b: boolean) { - includeImageDescriptions = b; -} -let startPlay: Date; -let fakeNarrationAborted: boolean = false; -let fakeNarrationTimer: number; -export let PageDuration: number; - -export function pause() { - if (currentPlaybackMode === PlaybackMode.AudioPaused) { - return; - } - pausePlaying(); - startPause = new Date(); - - // Note that neither music.pause() nor animations.PauseAnimations() check the state. - // If that changes, then this state setting might need attention. - setCurrentPlaybackMode(PlaybackMode.AudioPaused); -} - -// This pauses the current player without setting the "AudioPaused" state or setting the -// startPause timestamp. If this method is called when resumption is possible, the calling -// method must take care of these values (as in the pause method directly above). -// Note that there's no "stop" method on player, only a "pause" method. This method is -// used both when "pausing" the narration while viewing a page and when stopping narration -// when changing pages. -function pausePlaying() { - const player = getPlayer(); - if (segments && segments.length && player) { - // Before reporting duration, try to check that we really are playing. - // a separate report is sent if play ends. - if (player.currentTime > 0 && !player.paused && !player.ended) { - reportPlayDuration(); - } - player.pause(); - } -} - -export function hidingPage() { - pausePlaying(); // Doesn't set AudioPaused state. Caller sets NewPage state. - clearTimeout(pageNarrationCompleteTimer); -} - - // Roughly equivalent to BloomDesktop's AudioRecording::listen() function. - // As long as there is audio on the page, this method will play it. - export function playAllSentences(page: HTMLElement | null): void { - if (!page && !currentPlayPage) { - return; // this shouldn't happen - } - if (page) { - currentPlayPage = page; // Review: possibly redundant? Do all callers set currentPlayPage independently? - } - const mediaPlayer = getPlayer(); - if (mediaPlayer) { - mediaPlayer.pause(); - mediaPlayer.currentTime = 0; - } - - // Invalidate old ID, even if there's no new audio to play. - // (Deals with the case where you are on a page with audio, switch to a page without audio, then switch back to original page) - ++currentAudioSessionNum; - - fixHighlighting(); - - // Sorted into the order we want to play them, then reversed so we - // can more conveniently pop the next one to play from the end of the stack. - elementsToPlayConsecutivelyStack = sortAudioElements( - getPageAudioElements() - ).reverse(); - - const stackSize = elementsToPlayConsecutivelyStack.length; - if (stackSize === 0) { - // Nothing to play. Wait the standard amount of time anyway, in case we're autoadvancing. - if (PageNarrationComplete) { - pageNarrationCompleteTimer = window.setTimeout(() => { - PageNarrationComplete.raise(); - }, durationOfPagesWithoutNarration * 1000); - } - if (PlayCompleted) { - PlayCompleted.raise(); - } - return; - } - - const firstElementToPlay = elementsToPlayConsecutivelyStack[ - stackSize - 1 - ]; // Remember to pop it when you're done playing it. (i.e., in playEnded) - - setSoundAndHighlight(firstElementToPlay, true); - playCurrentInternal(); - return; - } - - let pageNarrationCompleteTimer: number; -// Indicates that the element should be highlighted. -const kEnableHighlightClass = "ui-enableHighlight"; -// Indicates that the element should NOT be highlighted. -// For example, some elements have highlighting prevented at this level -// because its content has been broken into child elements, only some of which show the highlight -const kDisableHighlightClass = "ui-disableHighlight"; -// Match space or   (\u00a0). Must have three or more in a row to match. -// Note: Multi whitespace text probably contains a bunch of   followed by a single normal space at the end. -const multiSpaceRegex = /[ \u00a0]{3,}/; -const multiSpaceRegexGlobal = new RegExp(multiSpaceRegex, "g"); -/** - * Finds and fixes any elements on the page that should have their audio-highlighting disabled. - */ -function fixHighlighting() { - // Note: Only relevant when playing by sentence (but note, this can make Record by Text Box -> Split or Record by Sentence, Play by Sentence) - // Play by Text Box highlights the whole paragraph and none of this really matters. - // (the span selector won't match anyway) - const audioElements = getPageAudioElements(); - audioElements.forEach(audioElement => { - // FYI, don't need to process the bloom-linebreak spans. Nothing bad happens, just unnecessary. - const matches = findAll( - "span:not(.bloom-linebreak)", - audioElement, - true - ); - matches.forEach(element => { - // Simple check to help ensure that elements that don't need to be modified will remain untouched. - // This doesn't consider whether text that shouldn't be highlighted is already in inside an - // element with highlight disabled, but that's ok. The code down the stack checks that. - const containsNonHighlightText = !!element.innerText.match( - multiSpaceRegex - ); - - if (containsNonHighlightText) { - fixHighlightingInNode(element, element); - } - }); - }); -} - -/** - * Recursively fixes the audio-highlighting within a node (whether element node or text node) - * @param node The node to recursively fix - * @param startingSpan The starting span, AKA the one that will receive .ui-audioCurrent in the future. - */ -function fixHighlightingInNode(node: Node, startingSpan: HTMLSpanElement) { - if ( - node.nodeType === Node.ELEMENT_NODE && - (node as Element).classList.contains(kDisableHighlightClass) - ) { - // No need to process bloom-highlightDisabled elements (they've already been processed) - return; - } else if (node.nodeType === Node.TEXT_NODE) { - // Leaf node. Fix the highlighting, then go back up the stack. - fixHighlightingInTextNode(node, startingSpan); - return; - } else { - // Recursive case - const childNodesCopy = Array.from(node.childNodes); // Make a copy because node.childNodes is being mutated - childNodesCopy.forEach(childNode => { - fixHighlightingInNode(childNode, startingSpan); - }); - } -} - -/** - * Analyzes a text node and fixes its highlighting. - */ -function fixHighlightingInTextNode( - textNode: Node, - startingSpan: HTMLSpanElement -) { - if (textNode.nodeType !== Node.TEXT_NODE) { - throw new Error( - "Invalid argument to fixMultiSpaceInTextNode: node must be a TextNode" - ); - } - - if (!textNode.nodeValue) { - return; - } - - // string.matchAll would be cleaner, but not supported in all browsers (in particular, FF60) - // Use RegExp.exec for greater compatibility. - multiSpaceRegexGlobal.lastIndex = 0; // RegExp.exec is stateful! Need to reset the state. - const matches: { - text: string; - startIndex: number; - endIndex: number; // the index of the first character to exclude - }[] = []; - let regexResult: RegExpExecArray | null; - while ( - (regexResult = multiSpaceRegexGlobal.exec( - textNode.nodeValue - )) != null - ) { - regexResult.forEach(matchingText => { - matches.push({ - text: matchingText, - startIndex: - multiSpaceRegexGlobal.lastIndex - - matchingText.length, - endIndex: multiSpaceRegexGlobal.lastIndex // the index of the first character to exclude - }); - }); - } - - // First, generate the new DOM elements with the fixed highlighting. - const newNodes: Node[] = []; - if (matches.length === 0) { - // No matches - newNodes.push(makeHighlightedSpan(textNode.nodeValue)); - } else { - let lastMatchEndIndex = 0; // the index of the first character to exclude of the last match - for (let i = 0; i < matches.length; ++i) { - const match = matches[i]; - - const preMatchText = textNode.nodeValue.slice( - lastMatchEndIndex, - match.startIndex - ); - lastMatchEndIndex = match.endIndex; - newNodes.push(makeHighlightedSpan(preMatchText)); - - newNodes.push(document.createTextNode(match.text)); - - if (i === matches.length - 1) { - const postMatchText = textNode.nodeValue.slice( - match.endIndex - ); - if (postMatchText) { - newNodes.push(makeHighlightedSpan(postMatchText)); - } - } - } - } - - // Next, replace the old DOM element with the new DOM elements - const oldNode = textNode; - if (oldNode.parentNode && newNodes && newNodes.length > 0) { - for (let i = 0; i < newNodes.length; ++i) { - const nodeToInsert = newNodes[i]; - oldNode.parentNode.insertBefore(nodeToInsert, oldNode); - } - - oldNode.parentNode.removeChild(oldNode); - - // We need to set ancestor's background back to transparent (instead of highlighted), - // and let each of the newNodes's styles control whether to be highlighted or transparent. - // If ancestor was highlighted but one of its new descendant nodes was transparent, - // all that would happen is the descendant would allow the ancestor's highlight color to show through, - // which doesn't achieve what we want :( - startingSpan.classList.add(kDisableHighlightClass); - } -} - -function makeHighlightedSpan(textContent: string) { - const newSpan = document.createElement("span"); - newSpan.classList.add(kEnableHighlightClass); - newSpan.appendChild(document.createTextNode(textContent)); - return newSpan; -} - - // Optional param is for use when 'playerPage' has NOT been initialized. -// Not using the optional param assumes 'playerPage' has been initialized -function getPageAudioElements(page?: HTMLElement): HTMLElement[] { - return [].concat.apply( - [], - getPagePlayableDivs(page).map(x => - findAll(".audio-sentence", x, true) - ) - ); -} - - // Returns all elements that match CSS selector {expr} as an array. -// Querying can optionally be restricted to {container}’s descendants -// If includeSelf is true, it includes both itself as well as its descendants. -// Otherwise, it only includes descendants. -// Also filters out imageDescriptions if we aren't supposed to be reading them. -function findAll( - expr: string, - container: HTMLElement, - includeSelf: boolean = false -): HTMLElement[] { - // querySelectorAll checks all the descendants - const allMatches: HTMLElement[] = [].slice.call( - (container || document).querySelectorAll(expr) - ); - - // Now check itself - if (includeSelf && container && container.matches(expr)) { - allMatches.push(container); - } - - return includeImageDescriptions - ? allMatches - : allMatches.filter( - match => !isImageDescriptionSegment(match) - ); -} - -function getPlayableDivs(container: HTMLElement) { - // We want to play any audio we have from divs the user can see. - // This is a crude test, but currently we always use display:none to hide unwanted languages. - return findAll(".bloom-editable", container).filter( - e => window.getComputedStyle(e).display !== "none" - ); -} - -// Optional param is for use when 'playerPage' has NOT been initialized. -// Not using the optional param assumes 'playerPage' has been initialized -function getPagePlayableDivs(page?: HTMLElement): HTMLElement[] { - return getPlayableDivs(page ? page : currentPlayPage!); -} - -export function play() { - if (currentPlaybackMode === PlaybackMode.AudioPlaying) { - return; // no change. - } - setCurrentPlaybackMode(PlaybackMode.AudioPlaying); - // I'm not sure how getPlayer() can return null/undefined, but have seen it happen - // typically when doing something odd like trying to go back from the first page. - if (segments.length && getPlayer()) { - if (elementsToPlayConsecutivelyStack.length) { - handlePlayPromise(getPlayer().play()); - - // Resuming play. Only currentStartTime needs to be adjusted, but originalStartTime shouldn't be changed. - audioPlayCurrentStartTime = new Date().getTime(); - } else { - // Pressing the play button in this case is triggering a replay of the current page, - // so we need to reset the highlighting. - playAllSentences(null); - return; - } - } - // adjust startPlay by the elapsed pause. This will cause fakePageNarrationTimedOut to - // start a new timeout if we are depending on it to fake PageNarrationComplete. - const pause = new Date().getTime() - startPause.getTime(); - startPlay = new Date(startPlay.getTime() + pause); - //console.log("paused for " + pause + " and adjusted start time to " + startPlay); - if (fakeNarrationAborted) { - // we already paused through the timeout for normal advance. - // This call (now we are not paused and have adjusted startPlay) - // will typically start a new timeout. If we are very close to - // the desired duration it may just raise the event at once. - // Either way we should get the event raised exactly once - // at very close to the right time, allowing for pauses. - fakeNarrationAborted = false; - fakePageNarrationTimedOut(currentPlayPage!); - } - // in case we're resuming play, we need a new timout when the current subelement is finished - highlightNextSubElement(currentAudioSessionNum); -} - - -function fakePageNarrationTimedOut(page: HTMLElement) { - if (currentPlaybackMode === PlaybackMode.AudioPaused) { - fakeNarrationAborted = true; - clearTimeout(fakeNarrationTimer); - return; - } - // It's possible we experienced one or more pauses and therefore this timeout - // happened too soon. In that case, startPlay will have been adjusted by - // the pauses, so we can detect that here and start a new timeout which will - // occur at the appropriately delayed time. - const duration = - (new Date().getTime() - startPlay.getTime()) / 1000; - if (duration < PageDuration - 0.01) { - // too soon; try again. - clearTimeout(fakeNarrationTimer); - fakeNarrationTimer = window.setTimeout( - () => fakePageNarrationTimedOut(page), - (PageDuration - duration) * 1000 - ); - return; - } - if (PageNarrationComplete) { - PageNarrationComplete.raise(page); - } -} - -// Figure out the total duration of the audio on the page. -// Currently has side effects of setting the current page and segments. -// I think that should be removed. -// An earlier version of this code (see narration.ts around November 2023) -// was designed to run asnychronously so that if we don't have audio -// durations in the file, it would try to get the actual duration of the audio -// from the server. However, comments indicated that this approach did not -// work in mobile apps, and bloompubs have now long shipped with the durations. -// So I decided to simplify. -export function computeDuration(page: HTMLElement): number { - currentPlayPage = page; - segments = getPageAudioElements(); - PageDuration = 0.0; - startPlay = new Date(); - //console.log("started play at " + startPlay); - // in case we are already paused (but did manual advance), start computing - // the pause duration from the beginning of this page. - startPause = startPlay; - if (segments.length === 0) { - PageDuration = durationOfPagesWithoutNarration; - if (PageDurationAvailable) { - PageDurationAvailable.raise(page); - } - // Since there is nothing to play, we will never get an 'ended' event - // from the player. If we are going to advance pages automatically, - // we need to raise PageNarrationComplete some other way. - // A timeout allows us to raise it after the arbitrary duration we have - // selected. The tricky thing is to allow it to be paused. - clearTimeout(fakeNarrationTimer); - fakeNarrationTimer = window.setTimeout( - () => fakePageNarrationTimedOut(page), - PageDuration * 1000 - ); - fakeNarrationAborted = false; - return PageDuration; - } - - segments.forEach((segment, index) => { - const attrDuration = segment.getAttribute( - "data-duration" - ); - if (attrDuration) { - // precomputed duration available, use it and go on. - PageDuration += parseFloat(attrDuration); - } - }); - if (PageDuration < durationOfPagesWithoutNarration) { - PageDuration = durationOfPagesWithoutNarration; - } - return PageDuration; -} - -export function pageHasAudio(page: HTMLElement): boolean { - return getPageAudioElements(page).length ? true : false; -} diff --git a/src/stories/index.tsx b/src/stories/index.tsx index 3c3fd0e0..bfaafdb0 100644 --- a/src/stories/index.tsx +++ b/src/stories/index.tsx @@ -200,10 +200,6 @@ AddBloomPlayerStory( "Activity/Choice activities from Bloom 5.4", "testbooks/Bloom5.4-activities/Bloom5.4-activities.htm" ); -AddBloomPlayerStory( - "Activity/Dragging", - "testbooks/Word%20Slider/Word%20Slider.htm" -); AddBloomPlayerStory( "Book with two audio sentences on cover", "https://s3.amazonaws.com/bloomharvest/namitaj%40chetana.org.in/78c7e561-ce24-4e5d-ad0a-6af141d9d0af/bloomdigital%2findex.htm"