Skip to content

Commit b87a776

Browse files
committed
Implement TTS reading out the article
1 parent 4da6a6b commit b87a776

File tree

7 files changed

+449
-12
lines changed

7 files changed

+449
-12
lines changed

app/src/gestalt/java/io/bimmergestalt/reader/carapp/CarApp.kt

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
package io.bimmergestalt.reader.carapp
22

33
import android.util.Log
4+
import com.google.gson.Gson
5+
import com.google.gson.JsonSyntaxException
46
import de.bmw.idrive.BMWRemoting
57
import de.bmw.idrive.BMWRemotingServer
68
import de.bmw.idrive.BaseBMWRemotingClient
9+
import io.bimmergestalt.idriveconnectkit.CDSProperty
710
import io.bimmergestalt.idriveconnectkit.IDriveConnection
811
import io.bimmergestalt.idriveconnectkit.Utils.rhmi_setResourceCached
9-
import io.bimmergestalt.idriveconnectkit.android.CarAppResources
1012
import io.bimmergestalt.idriveconnectkit.android.IDriveConnectionStatus
1113
import io.bimmergestalt.idriveconnectkit.android.security.SecurityAccess
1214
import io.bimmergestalt.idriveconnectkit.rhmi.RHMIApplication
@@ -18,36 +20,44 @@ import io.bimmergestalt.idriveconnectkit.rhmi.RHMIState
1820
import io.bimmergestalt.reader.carapp.views.FeedView
1921
import io.bimmergestalt.reader.carapp.views.HomeView
2022
import io.bimmergestalt.reader.carapp.views.ReadView
23+
import io.bimmergestalt.reader.carapp.views.ReadoutView
2124
import me.ash.reader.domain.service.RssService
2225

2326
const val TAG = "ReaderGestalt"
2427
class CarApp(val iDriveConnectionStatus: IDriveConnectionStatus, securityAccess: SecurityAccess,
25-
val carAppResources: CarAppResources, val rssService: RssService
28+
val carAppResources: CarAppSharedAssetResources, val rssService: RssService
2629
) {
2730

2831
val carConnection: BMWRemotingServer
2932
val carApp: RHMIApplication
33+
val readoutController: ReadoutController
3034
val model: Model = Model()
3135
val homeView: HomeView
3236
val feedView: FeedView
3337
val readView: ReadView
38+
val readoutView: ReadoutView
3439

3540
init {
3641
Log.i(TAG, "Starting connecting to car")
3742
val carappListener = CarAppListener()
3843
carConnection = IDriveConnection.getEtchConnection(iDriveConnectionStatus.host ?: "127.0.0.1", iDriveConnectionStatus.port ?: 8003, carappListener)
39-
val appCert = carAppResources.getAppCertificate(iDriveConnectionStatus.brand ?: "")?.readBytes()
44+
val appCert = carAppResources.getAppCertificate(iDriveConnectionStatus.brand ?: "").readBytes()
4045
val sas_challenge = carConnection.sas_certificate(appCert)
4146
val sas_response = securityAccess.signChallenge(challenge = sas_challenge)
4247
carConnection.sas_login(sas_response)
4348

4449
carApp = createRhmiApp()
50+
readoutController = ReadoutController.build(carApp, "News")
4551
val destStateId = carApp.components.values.filterIsInstance<RHMIComponent.EntryButton>().first().getAction()?.asHMIAction()?.target!!
4652
homeView = HomeView(carApp.states[destStateId] as RHMIState, rssService, model)
4753
feedView = FeedView(carApp.states[homeView.getFeedButtonDest()]!!, rssService, model)
4854
readView = ReadView(carApp.states[homeView.getEntryListDest()] as RHMIState.ToolbarState, model)
55+
readoutView = ReadoutView(carApp.states[readView.getReadoutDest()] as RHMIState.ToolbarState, readoutController, model)
4956

5057
initWidgets()
58+
59+
createCds()
60+
5161
Log.i(TAG, "CarApp running")
5262
}
5363

@@ -57,6 +67,8 @@ class CarApp(val iDriveConnectionStatus: IDriveConnectionStatus, securityAccess:
5767
carConnection.rhmi_setResourceCached(rhmiHandle, BMWRemoting.RHMIResourceType.DESCRIPTION, carAppResources.getUiDescription())
5868
carConnection.rhmi_setResourceCached(rhmiHandle, BMWRemoting.RHMIResourceType.TEXTDB, carAppResources.getTextsDB("common"))
5969
carConnection.rhmi_setResourceCached(rhmiHandle, BMWRemoting.RHMIResourceType.IMAGEDB, carAppResources.getImagesDB(iDriveConnectionStatus.brand ?: "common"))
70+
carConnection.rhmi_setResourceCached(rhmiHandle, BMWRemoting.RHMIResourceType.TEXTDB, carAppResources.getSharedTextsDB("common"))
71+
carConnection.rhmi_setResourceCached(rhmiHandle, BMWRemoting.RHMIResourceType.IMAGEDB, carAppResources.getSharedImagedDB(iDriveConnectionStatus.brand ?: "common"))
6072
carConnection.rhmi_initialize(rhmiHandle)
6173

6274
// register for events from the car
@@ -71,13 +83,24 @@ class CarApp(val iDriveConnectionStatus: IDriveConnectionStatus, securityAccess:
7183
}
7284
}
7385

86+
private fun createCds() {
87+
synchronized(carConnection) {
88+
val cdsHandle = carConnection.cds_create()
89+
for (prop in listOf(CDSProperty.HMI_TTS)) {
90+
carConnection.cds_addPropertyChangedEventHandler(cdsHandle, prop.propertyName, prop.ident.toString(), 200)
91+
carConnection.cds_getPropertyAsync(cdsHandle, prop.ident.toString(), prop.propertyName)
92+
}
93+
}
94+
}
95+
7496
private fun initWidgets() {
7597
carApp.components.values.filterIsInstance<RHMIComponent.EntryButton>().forEach {
7698
it.getAction()?.asHMIAction()?.getTargetModel()?.asRaIntModel()?.value = homeView.state.id
7799
}
78100
homeView.initWidgets()
79101
feedView.initWidgets()
80102
readView.initWidgets()
103+
readoutView.initWidgets()
81104
}
82105

83106
fun onDestroy() {
@@ -117,5 +140,18 @@ class CarApp(val iDriveConnectionStatus: IDriveConnectionStatus, securityAccess:
117140
Log.e(TAG, "Received exception while handling rhmi_onHmiEvent", e)
118141
}
119142
}
143+
144+
override fun cds_onPropertyChangedEvent(handle: Int?, ident: String?, propertyName: String?, propertyValue: String?) {
145+
propertyValue ?: return
146+
if (propertyName == "hmi.tts") {
147+
try {
148+
val hmiTTS = Gson().fromJson(propertyValue, HMITTS::class.java)
149+
readoutController.onTTSEvent(hmiTTS.TTSState)
150+
} catch (e: JsonSyntaxException) {
151+
Log.e(TAG, "Received unexpected hmiTTS $propertyValue", e)
152+
readoutController.onTTSEvent(TTSState(null, null, null, e.toString(), null))
153+
}
154+
}
155+
}
120156
}
121157
}

app/src/gestalt/java/io/bimmergestalt/reader/carapp/CarAppService.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ class CarAppService: Service() {
7474
app = CarApp(
7575
iDriveConnectionStatus,
7676
securityAccess,
77-
CarAppAssetResources(applicationContext, "news"),
77+
CarAppSharedAssetResources(applicationContext, "news"),
7878
rssService
7979
)
8080
}
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package io.bimmergestalt.reader.carapp
2+
3+
import android.content.Context
4+
import io.bimmergestalt.idriveconnectkit.android.CarAppAssetResources
5+
import java.io.InputStream
6+
import java.util.Locale
7+
8+
class CarAppSharedAssetResources(context: Context, name: String): CarAppAssetResources(context, name) {
9+
companion object {
10+
const val IMG_BUILTIN_PREV_PAGE = 55002
11+
const val IMG_BUILTIN_NEXT_PAGE = 55001
12+
const val IMG_BUILTIN_STATUS_ACTIVE = 55009
13+
const val IMG_BUILTIN_PREV_MESSAGE = 55003
14+
const val IMG_BUILTIN_NEXT_MESSAGE = 55004
15+
16+
const val IMG_PLAY = 58001
17+
const val IMG_PAUSE = 58002
18+
const val IMG_PREV_PARAGRAPH = 58003
19+
const val IMG_NEXT_PARAGRAPH = 58004
20+
const val IMG_PREV_MESSAGE = 58005
21+
const val IMG_NEXT_MESSAGE = 58006
22+
const val IMG_STATUS_ACTIVE = 58007
23+
const val IMG_STATUS_PAUSED = 58008
24+
const val IMG_STATUS_STOPPED = 58009
25+
26+
const val TXT_PLAY = 59001
27+
const val TXT_PAUSE = 59002
28+
const val TXT_PREV_PARAGRAPH = 59003
29+
const val TXT_NEXT_PARAGRAPH = 59004
30+
const val TXT_PREV_MESSAGE = 59005
31+
const val TXT_NEXT_MESSAGE = 59006
32+
const val TXT_STATUS_ACTIVE = 59007
33+
const val TXT_STATUS_PAUSED = 59008
34+
const val TXT_STATUS_STOPPED = 59009
35+
const val TXT_TEXT_TO_SPEECH = 59010
36+
const val TXT_BEGINNING = 59011
37+
const val TXT_LINK_HTTP = 59012
38+
const val TXT_LINK_FTP = 59013
39+
const val TXT_LINK = 59014
40+
}
41+
fun getSharedImagedDB(brand: String): InputStream? {
42+
return loadFile("carapplications/$name/rhmi/${brand.lowercase(Locale.ROOT)}/images_shared.zip") ?:
43+
loadFile("carapplications/$name/rhmi/common/images_shared.zip")
44+
}
45+
fun getSharedTextsDB(brand: String): InputStream? {
46+
return loadFile("carapplications/$name/rhmi/${brand.lowercase(Locale.ROOT)}/texts_shared.zip") ?:
47+
loadFile("carapplications/$name/rhmi/common/texts_shared.zip")
48+
}
49+
}
Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
package io.bimmergestalt.reader.carapp
2+
3+
import android.util.Log
4+
import io.bimmergestalt.idriveconnectkit.rhmi.RHMIApplication
5+
import io.bimmergestalt.idriveconnectkit.rhmi.RHMIEvent
6+
import io.bimmergestalt.idriveconnectkit.rhmi.RHMIModel
7+
import kotlinx.coroutines.flow.MutableStateFlow
8+
import kotlinx.coroutines.flow.combine
9+
import kotlin.math.max
10+
import kotlin.math.min
11+
12+
13+
data class HMITTS( // the json decoder object
14+
val TTSState: TTSState
15+
)
16+
data class TTSState( // the actual state
17+
val state: Int?,
18+
val currentblock: Int?,
19+
val blocks: Int?,
20+
val type: String?,
21+
val languageavailable: Int?,
22+
) {
23+
val stateName: ReadoutState
24+
get() = ReadoutState.fromValue(state)
25+
26+
override fun toString(): String {
27+
return "$type ${stateName.name} - $currentblock/$blocks"
28+
}
29+
}
30+
enum class ReadoutState(val value: Int) { // enum name for the state int
31+
UNDEFINED(0),
32+
IDLE(1),
33+
PAUSED(2),
34+
ACTIVE(3),
35+
BUSY(4);
36+
37+
companion object {
38+
fun fromValue(value: Int?): ReadoutState {
39+
return values().firstOrNull { it.value == value } ?: UNDEFINED
40+
}
41+
}
42+
}
43+
enum class ReadoutCommand(val value: String) {
44+
PAUSE("STR_READOUT_PAUSE"),
45+
STOP("STR_READOUT_STOP"),
46+
PREV_BLOCK("STR_READOUT_PREV_BLOCK"),
47+
NEXT_BLOCK("STR_READOUT_NEXT_BLOCK"),
48+
RESTART("STR_READOUT_JUMP_TO_BEGIN"),
49+
}
50+
51+
class ReadoutController(val name: String, val speechEvent: RHMIEvent.ActionEvent, val commandEvent: RHMIEvent.ActionEvent) {
52+
val speechList = speechEvent.getAction()?.asLinkAction()?.getLinkModel()?.asRaListModel()!!
53+
val commandList = commandEvent.getAction()?.asLinkAction()?.getLinkModel()?.asRaListModel()!!
54+
55+
companion object {
56+
fun build(app: RHMIApplication, name: String): ReadoutController {
57+
val events = app.events.values.filterIsInstance<RHMIEvent.ActionEvent>().filter {
58+
it.getAction()?.asLinkAction()?.actionType == "readout"
59+
}
60+
if (events.size != 2) {
61+
throw IllegalArgumentException("UI Description is missing 2 readout events")
62+
}
63+
return ReadoutController(name, events[0], events[1])
64+
}
65+
}
66+
67+
var desiredState = ReadoutState.IDLE
68+
val activeState = MutableStateFlow(ReadoutState.IDLE) // whether we are currently talking
69+
private var currentState = TTSState(0, null, null, null, null)
70+
val debugText = MutableStateFlow(currentState.toString())
71+
val isActive: Boolean
72+
get() = currentState.type == name &&
73+
(currentState.stateName == ReadoutState.ACTIVE || currentState.stateName == ReadoutState.BUSY)
74+
75+
private val lineIndex = MutableStateFlow(0)
76+
private var nextLineIndex = -1 // next line index to read at the next IDLE state
77+
private var lines = MutableStateFlow(emptyList<String>())
78+
val currentLine = lines.combine(lineIndex) { lines, i ->
79+
lines.getOrNull(i) ?: ""
80+
}
81+
82+
fun onTTSEvent(ttsState: TTSState) {
83+
currentState = ttsState
84+
debugText.value = currentState.toString()
85+
Log.d(TAG, "TTSEvent: currentState:${ttsState.stateName} currentName:${ttsState.type} currentBlock:${ttsState.currentblock}/${ttsState.blocks}")
86+
87+
if (desiredState == ReadoutState.ACTIVE && ttsState.stateName == ReadoutState.IDLE) {
88+
if (nextLineIndex >= 0) {
89+
lineIndex.value = nextLineIndex
90+
readLine()
91+
return // don't update activeState
92+
} else {
93+
desiredState = ReadoutState.IDLE // automatically stop
94+
}
95+
}
96+
// we aren't automatically continuing on, update the public activeState
97+
activeState.value = if (ttsState.type == name) {
98+
when (ttsState.stateName) {
99+
ReadoutState.ACTIVE -> ReadoutState.ACTIVE
100+
ReadoutState.BUSY -> ReadoutState.ACTIVE
101+
ReadoutState.PAUSED -> ReadoutState.PAUSED
102+
else -> ReadoutState.IDLE
103+
}
104+
} else { // some other TTS app is speaking
105+
ReadoutState.IDLE
106+
}
107+
}
108+
109+
fun readLines(lines: List<String>) {
110+
loadLines(lines)
111+
play()
112+
}
113+
114+
fun loadLines(lines: List<String>) {
115+
this.lines.value = lines
116+
this.lineIndex.value = 0
117+
nextLineIndex = 0
118+
desiredState = ReadoutState.IDLE
119+
}
120+
121+
fun play() {
122+
desiredState = ReadoutState.ACTIVE
123+
readLine()
124+
}
125+
126+
127+
fun prevLine() {
128+
nextLineIndex = max(0, lineIndex.value - 1)
129+
lineIndex.value = nextLineIndex
130+
_stop()
131+
}
132+
133+
fun nextLine() {
134+
nextLineIndex = min(lines.value.size - 1, lineIndex.value + 1)
135+
lineIndex.value = nextLineIndex
136+
_stop()
137+
}
138+
139+
private fun readLine() {
140+
val line = lines.value.getOrNull(lineIndex.value) ?: ""
141+
val data = RHMIModel.RaListModel.RHMIListConcrete(2)
142+
data.addRow(arrayOf(line, name))
143+
Log.d(TAG, "Starting readout from $name: ${data[0][0]}")
144+
speechList.setValue(data, 0, 1, 1)
145+
speechEvent.triggerEvent()
146+
147+
// cue the next line to play
148+
if (lineIndex.value < lines.value.size - 1) {
149+
nextLineIndex = lineIndex.value + 1
150+
} else {
151+
nextLineIndex = -1
152+
}
153+
}
154+
155+
fun pause() {
156+
desiredState = ReadoutState.PAUSED
157+
158+
Log.d(TAG, "Pausing $name readout")
159+
val data = RHMIModel.RaListModel.RHMIListConcrete(2).apply {
160+
addRow(arrayOf(ReadoutCommand.PAUSE.value, name))
161+
}
162+
commandList.setValue(data, 0, 1, 1)
163+
commandEvent.triggerEvent()
164+
}
165+
fun stop() {
166+
desiredState = ReadoutState.IDLE
167+
_stop()
168+
}
169+
private fun _stop() {
170+
Log.d(TAG, "Cancelling $name readout")
171+
val data = RHMIModel.RaListModel.RHMIListConcrete(2).apply {
172+
addRow(arrayOf(ReadoutCommand.STOP.value, name))
173+
}
174+
commandList.setValue(data, 0, 1, 1)
175+
commandEvent.triggerEvent()
176+
}
177+
}

0 commit comments

Comments
 (0)