Text-to-speech (feature branch)

app/build.gradle

@@ -183,6 +183,7 @@ dependencies {
     implementation libs.material
     implementation libs.appcompat
     implementation libs.core.ktx
+    implementation libs.concurrent.futures
     implementation libs.browser
     implementation libs.constraintlayout
     implementation libs.fragment.ktx
@@ -213,6 +214,10 @@ dependencies {
     annotationProcessor libs.androidx.room.compiler
     ksp libs.androidx.room.compiler
     implementation libs.androidx.room.ktx
+    implementation libs.media3.common
+    implementation libs.media3.exoplayer
+    implementation libs.media3.session
+    implementation libs.media3.ui
 
     // For language detection during editing
     prodImplementation libs.com.google.mlkit.language.id

app/src/main/AndroidManifest.xml

@@ -50,6 +50,9 @@
     <uses-permission android:name="android.permission.ACCESS_COARSE_LOCATION" android:required="false" />
     <uses-permission android:name="android.permission.ACCESS_FINE_LOCATION" android:required="false" />
 
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE" android:required="false" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PLAYBACK" android:required="false" />
+
     <queries>
         <intent>
             <action android:name="android.intent.action.VIEW" />
@@ -447,5 +450,15 @@
             </intent-filter>
         </service>
 
+        <service
+            android:name=".page.tts.PlaybackService"
+            android:foregroundServiceType="mediaPlayback"
+            android:exported="true"
+            android:permission="android.permission.FOREGROUND_SERVICE_MEDIA_PLAYBACK">
+            <intent-filter>
+                <action android:name="androidx.media3.session.MediaSessionService"/>
+            </intent-filter>
+        </service>
+
     </application>
 </manifest>

app/src/main/java/org/wikipedia/bridge/JavaScriptActionHandler.kt

@@ -205,6 +205,27 @@ object JavaScriptActionHandler {
                 "})();"
     }
 
+    fun getSpokenFileName(): String {
+        return "(function() {" +
+                "  let spokenDiv = document.querySelector('.spoken-wikipedia');" +
+                "  let spokenAudio = spokenDiv.querySelector('audio');" +
+                "  let spokenSources = spokenDiv.querySelectorAll('source');" +
+                "  return spokenSources[0].getAttribute('src');" +
+                "})();"
+    }
+
+    fun getSectionContents(): String {
+        return "(function() {" +
+                "  let sections = document.querySelectorAll('section');" +
+                "  let section = sections[0].cloneNode(true);" +
+                "  let elements = section.querySelectorAll('style,.IPA,.mw-ref,.hatnote,.pcs-collapse-table-container');" +
+                "  for (let i = 0; i < elements.length; i++) {" +
+                "    elements[i].remove();" +
+                "  }" +
+                "  return section.innerText;" +
+                "})();"
+    }
+
     @Serializable
     class ImageHitInfo(val left: Float = 0f, val top: Float = 0f, val width: Float = 0f, val height: Float = 0f,
                        val src: String = "")

app/src/main/java/org/wikipedia/page/PageActivity.kt

@@ -64,6 +64,7 @@ import org.wikipedia.notifications.AnonymousNotificationHelper
 import org.wikipedia.notifications.NotificationActivity
 import org.wikipedia.page.linkpreview.LinkPreviewDialog
 import org.wikipedia.page.tabs.TabActivity
+import org.wikipedia.page.tts.Tts
 import org.wikipedia.readinglist.ReadingListActivity
 import org.wikipedia.readinglist.ReadingListMode
 import org.wikipedia.search.SearchActivity

app/src/main/java/org/wikipedia/page/PageFragment.kt

@@ -5,6 +5,7 @@ import android.app.Activity
 import android.content.Context
 import android.content.Intent
 import android.content.res.Configuration
+import android.icu.text.BreakIterator
 import android.net.Uri
 import android.os.Bundle
 import android.view.ActionMode
@@ -25,6 +26,7 @@ import androidx.core.animation.doOnEnd
 import androidx.core.app.ActivityCompat
 import androidx.core.app.ActivityOptionsCompat
 import androidx.core.view.forEach
+import androidx.core.view.isVisible
 import androidx.core.widget.TextViewCompat
 import androidx.fragment.app.Fragment
 import androidx.lifecycle.lifecycleScope
@@ -91,6 +93,9 @@ import org.wikipedia.page.references.PageReferences
 import org.wikipedia.page.references.ReferenceDialog
 import org.wikipedia.page.shareafact.ShareHandler
 import org.wikipedia.page.tabs.Tab
+import org.wikipedia.page.tts.NarrationPopupView
+import org.wikipedia.page.tts.PlaybackService
+import org.wikipedia.page.tts.Tts
 import org.wikipedia.places.PlacesActivity
 import org.wikipedia.readinglist.LongPressMenu
 import org.wikipedia.readinglist.ReadingListBehaviorsUtil
@@ -105,6 +110,7 @@ import org.wikipedia.util.FeedbackUtil
 import org.wikipedia.util.ImageUrlUtil
 import org.wikipedia.util.ResourceUtil
 import org.wikipedia.util.ShareUtil
+import org.wikipedia.util.StringUtil
 import org.wikipedia.util.ThrowableUtil
 import org.wikipedia.util.UriUtil
 import org.wikipedia.util.log.L
@@ -117,6 +123,7 @@ import org.wikipedia.watchlist.WatchlistViewModel
 import org.wikipedia.wiktionary.WiktionaryDialog
 import java.time.Duration
 import java.time.Instant
+import java.util.Locale
 
 class PageFragment : Fragment(), BackPressedHandler, CommunicationBridge.CommunicationBridgeListener, ThemeChooserDialog.Callback,
     ReferenceDialog.Callback, WiktionaryDialog.Callback, WatchlistExpiryDialog.Callback {
@@ -193,6 +200,29 @@ class PageFragment : Fragment(), BackPressedHandler, CommunicationBridge.Communi
     val isLoading get() = bridge.isLoading
     val leadImageEditLang get() = leadImagesHandler.callToActionEditLang
 
+    val checkTtsServiceRunnable = TtsCheckRunnable()
+
+    inner class TtsCheckRunnable() : Runnable {
+        override fun run() {
+            if (!isAdded) {
+                return
+            }
+
+            if (PlaybackService.isRunning) {
+                if (!binding.speechButton.isVisible) {
+                    binding.speechButton.show()
+                }
+            } else {
+                if (binding.speechButton.isVisible) {
+                    binding.speechButton.hide()
+                }
+            }
+
+            binding.speechButton.postDelayed(checkTtsServiceRunnable, 1000)
+        }
+    }
+
+
     override fun onCreateView(inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle?): View {
         _binding = FragmentPageBinding.inflate(inflater, container, false)
         webView = binding.pageWebView
@@ -228,6 +258,12 @@ class PageFragment : Fragment(), BackPressedHandler, CommunicationBridge.Communi
             }
         }
 
+        binding.speechButton.setOnClickListener {
+            Tts.currentPageTitle?.let { title ->
+                NarrationPopupView(requireActivity()).show(binding.speechButton, title)
+            }
+        }
+
         bottomBarHideHandler = ViewHideHandler(binding.pageActionsTabContainer, null, Gravity.BOTTOM, updateElevation = false) { false }
         bottomBarHideHandler.setScrollView(webView)
         bottomBarHideHandler.enabled = Prefs.readingFocusModeEnabled
@@ -253,6 +289,8 @@ class PageFragment : Fragment(), BackPressedHandler, CommunicationBridge.Communi
         if (shouldLoadFromBackstack(activity) || savedInstanceState != null) {
             reloadFromBackstack()
         }
+
+        binding.speechButton.postDelayed(checkTtsServiceRunnable, 1000)
     }
 
     override fun onSaveInstanceState(outState: Bundle) {
@@ -1477,6 +1515,67 @@ class PageFragment : Fragment(), BackPressedHandler, CommunicationBridge.Communi
             articleInteractionEvent?.logForwardClick()
             metricsPlatformArticleEventToolbarInteraction.logForwardClick()
         }
+
+        override fun onNarrateSelected() {
+            bridge.evaluate(JavaScriptActionHandler.getSpokenFileName()) { result ->
+                var spokenUrl = if (result == null || result == "null") "" else result
+                if (spokenUrl.length > 2 && spokenUrl.startsWith("\"") && spokenUrl.endsWith("\"")) {
+                    spokenUrl = spokenUrl.substring(1, spokenUrl.length - 1)
+                }
+                if (spokenUrl.isNotEmpty()) {
+                    speakFromUrl(UriUtil.resolveProtocolRelativeUrl(spokenUrl))
+                } else {
+                    speakFromTts()
+                }
+            }
+        }
+
+        private fun speakFromUrl(url: String) {
+            Tts.start(requireActivity(), title, url, listOf())
+        }
+
+        private fun speakFromTts() {
+            bridge.evaluate(JavaScriptActionHandler.getSectionContents()) { result ->
+                var text = if (result == null || result == "null") "" else result
+                if (text.length > 2 && text.startsWith("\"") && text.endsWith("\"")) {
+                    text = text.substring(1, text.length - 1)
+                }
+
+                // massage the text a bit further
+                text = text.replace("\\n", "\n").replace("\\\"", "\"").replace("\\'", "'")
+
+                text = StringUtil.fromHtml(title?.displayText)
+                    .toString() + "\n\n" + title?.description + "\n\n" + text
+
+
+                val sentences = mutableListOf<String>()
+                var sentence = ""
+
+                val iterator = BreakIterator.getSentenceInstance(Locale.getDefault())
+                iterator.setText(text)
+                var start: Int = iterator.first()
+                if (start != BreakIterator.DONE) {
+                    var end: Int = iterator.next()
+                    while (end != BreakIterator.DONE) {
+                        val chunk = text.substring(start, end)
+                        sentence += "$chunk "
+                        // sentences should be at least 32 characters long.
+                        // TODO: limit size of sentences to 1024 characters?
+                        if (sentence.length + chunk.length > 32) {
+                            sentences.add(sentence)
+                            sentence = ""
+                        }
+                        start = end
+                        end = iterator.next()
+                    }
+                }
+                if (sentence.isNotEmpty()) {
+                    sentences.add(sentence)
+                }
+
+                Tts.start(requireActivity(), title, "", sentences)
+            }
+        }
     }
 
     companion object {

app/src/main/java/org/wikipedia/page/action/PageActionItem.kt

@@ -82,6 +82,11 @@ enum class PageActionItem constructor(val id: Int,
         override fun select(cb: Callback) {
             cb.onViewOnMapSelected()
         }
+    },
+    NARRATE(14, R.id.page_narrate, R.string.action_item_narrate, R.drawable.ic_volume_up, false) {
+        override fun select(cb: Callback) {
+            cb.onNarrateSelected()
+        }
     };
 
     abstract fun select(cb: Callback)
@@ -108,6 +113,7 @@ enum class PageActionItem constructor(val id: Int,
         fun onEditArticleSelected()
         fun onViewOnMapSelected()
         fun forwardClick()
+        fun onNarrateSelected()
     }
 
     companion object {

app/src/main/java/org/wikipedia/page/tts/NarrationPopupView.kt

@@ -0,0 +1,113 @@
+package org.wikipedia.page.tts
+
+import android.content.Context
+import android.content.Intent
+import android.graphics.Color
+import android.os.Bundle
+import android.view.Gravity
+import android.view.LayoutInflater
+import android.view.View
+import android.view.ViewGroup
+import android.widget.FrameLayout
+import android.widget.PopupWindow
+import androidx.core.view.doOnDetach
+import androidx.core.view.isVisible
+import androidx.core.widget.PopupWindowCompat
+import androidx.media3.session.SessionCommand
+import org.wikipedia.R
+import org.wikipedia.databinding.ViewNarrationPopupBinding
+import org.wikipedia.page.PageTitle
+import org.wikipedia.util.DimenUtil
+import org.wikipedia.util.StringUtil
+import org.wikipedia.views.ViewUtil
+import androidx.core.graphics.drawable.toDrawable
+
+class NarrationPopupView(context: Context) : FrameLayout(context) {
+
+    private var binding = ViewNarrationPopupBinding.inflate(LayoutInflater.from(context), this, true)
+    private var popupWindowHost: PopupWindow? = null
+
+    fun show(anchorView: View, pageTitle: PageTitle) {
+        popupWindowHost = PopupWindow(this, ViewGroup.LayoutParams.WRAP_CONTENT,
+                ViewGroup.LayoutParams.WRAP_CONTENT, true)
+        popupWindowHost?.let {
+            it.setBackgroundDrawable(Color.TRANSPARENT.toDrawable())
+            PopupWindowCompat.setOverlapAnchor(it, true)
+            it.showAsDropDown(anchorView, 0, anchorView.height - DimenUtil.roundedDpToPx(48f*4), Gravity.END or Gravity.BOTTOM)
+        }
+
+        anchorView.doOnDetach {
+            dismissPopupWindowHost()
+        }
+
+        binding.articleTitle.text = StringUtil.fromHtml(pageTitle.displayText)
+        if (pageTitle.thumbUrl.isNullOrEmpty()) {
+            binding.articleThumbnail.isVisible = false
+        } else {
+            binding.articleThumbnail.isVisible = true
+            ViewUtil.loadImage(binding.articleThumbnail, pageTitle.thumbUrl)
+        }
+        updateSpeedButtons()
+        updatePlayPauseButton()
+
+        binding.seekBackButton.setOnClickListener {
+            Tts.mediaController?.sendCustomCommand(SessionCommand(PlaybackService.CUSTOM_COMMAND_REWIND_SEC, Bundle()), Bundle())
+            updatePlayPauseButton()
+        }
+
+        binding.seekForwardButton.setOnClickListener {
+            Tts.mediaController?.sendCustomCommand(SessionCommand(PlaybackService.CUSTOM_COMMAND_FORWARD_SEC, Bundle()), Bundle())
+            updatePlayPauseButton()
+        }
+
+        binding.playPauseButton.setOnClickListener {
+            Tts.mediaController?.let {
+                if (it.isPlaying) {
+                    it.pause()
+                } else {
+                    it.play()
+                }
+            }
+            updatePlayPauseButton()
+        }
+
+        binding.decreaseSpeedButton.setOnClickListener {
+            PlaybackService.speechRate -= 0.1f
+            Tts.mediaController?.setPlaybackSpeed(PlaybackService.speechRate)
+            updateSpeedButtons()
+        }
+
+        binding.increaseSpeedButton.setOnClickListener {
+            PlaybackService.speechRate += 0.1f
+            Tts.mediaController?.setPlaybackSpeed(PlaybackService.speechRate)
+            updateSpeedButtons()
+        }
+
+        binding.stopButton.setOnClickListener {
+
+            context.stopService(Intent(context, PlaybackService::class.java))
+            Tts.cleanup()
+
+            dismissPopupWindowHost()
+        }
+    }
+
+    private fun dismissPopupWindowHost() {
+        popupWindowHost?.let {
+            it.dismiss()
+            popupWindowHost = null
+        }
+    }
+
+    private fun updatePlayPauseButton() {
+        Tts.mediaController?.let {
+            binding.playPauseButton.setImageResource(if (it.isPlaying) R.drawable.ic_pause_black_24dp else R.drawable.ic_play_arrow_black_24dp)
+        }
+    }
+
+    private fun updateSpeedButtons() {
+        binding.decreaseSpeedButton.isEnabled = PlaybackService.speechRate > 0.1f
+        binding.increaseSpeedButton.isEnabled = PlaybackService.speechRate < 2.0f
+        binding.speedText.text = String.format("%.1fx", PlaybackService.speechRate)
+    }
+}