Skip to content

Commit

Permalink
Data Collection: Add command counts
Browse files Browse the repository at this point in the history 
Also, update TomlKt, and ignore unknown keys in data adapter
  • Loading branch information
@gdude2002
gdude2002 committed Aug 17, 2024
1 parent 469b40d commit 474a161
Show file tree
Hide file tree
Showing 5 changed files with 157 additions and 54 deletions.
4 changes: 2 additions & 2 deletions gradle/libs.versions.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ sentry = "7.12.0"
slf4j = "2.0.13"
time4j-base = "5.9.4"
time4j-tzdata = "5.0-2024a"
toml = "0.2.0"
toml = "0.4.0"

[libraries]
commons-validator = { module = "commons-validator:commons-validator", version.ref = "commons-validator" }
Expand Down Expand Up @@ -85,7 +85,7 @@ sentry = { module = "io.sentry:sentry", version.ref = "sentry" }
slf4j = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" }
time4j-base = { module = "net.time4j:time4j-base", version.ref = "time4j-base" }
time4j-tzdata = { module = "net.time4j:time4j-tzdata", version.ref = "time4j-tzdata" }
toml = { module = "net.peanuuutz:tomlkt", version.ref = "toml" }
toml = { module = "net.peanuuutz.tomlkt:tomlkt", version.ref = "toml" }

[bundles]
commons = ["commons-validator"]
Expand Down
12 changes: 12 additions & 0 deletions kord-extensions/src/main/kotlin/dev/kordex/core/_Properties.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,18 @@ public val COLLECTION_STATE_LOCATION: String by lazy {
?: "./data/data-collection.properties"
}

/**
* Data collection UUID, if you need to specify one instead of having the storage system take care of it.
*
* Must be a valid UUID.
*/
public val DATA_COLLECTION_UUID: UUID? by lazy {
(
System.getProperties()["dataCollectionUUID"] as? String
?: envOrNull("DATA_COLLECTION_UUID")
)?.let { UUID.fromString(it) }
}

/**
* Data collection setting, defaulting to Standard if not set.
*
Expand Down
182 changes: 136 additions & 46 deletions kord-extensions/src/main/kotlin/dev/kordex/core/datacollection/DataCollector.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ import dev.kord.gateway.Intents
import dev.kordex.core.*
import dev.kordex.core.annotations.InternalAPI
import dev.kordex.core.builders.ExtensibleBotBuilder
import dev.kordex.core.commands.application.ApplicationCommandRegistry
import dev.kordex.core.commands.application.DefaultApplicationCommandRegistry
import dev.kordex.core.commands.chat.ChatCommandRegistry
import dev.kordex.core.koin.KordExKoinComponent
import dev.kordex.core.storage.StorageType
import dev.kordex.core.storage.StorageUnit
Expand All @@ -36,7 +39,6 @@ import java.io.IOException
import java.nio.file.Files
import java.util.*
import kotlin.time.Duration.Companion.minutes
import kotlin.time.Duration.Companion.seconds

@OptIn(InternalAPI::class)
@Suppress("StringLiteralDuplication", "MagicNumber")
Expand All @@ -45,6 +47,10 @@ public class DataCollector(public val level: DataCollection) : KordExKoinCompone

private val bot: ExtensibleBot by inject()
private val settings: ExtensibleBotBuilder by inject()
private val chatCommands: ChatCommandRegistry by inject()
private val _applicationCommands: ApplicationCommandRegistry by inject()

private val applicationCommands get() = _applicationCommands as? DefaultApplicationCommandRegistry

private val logger = KotlinLogging.logger { }
private val scheduler = Scheduler()
Expand All @@ -60,18 +66,26 @@ public class DataCollector(public val level: DataCollection) : KordExKoinCompone

internal suspend fun migrate() {
val props = loadOldState()
val current = getState()

if (props != null) {
logger.info { "Migrating from $COLLECTION_STATE_LOCATION to storage units..." }
if (DATA_COLLECTION_UUID == null) {
logger.info { "Migrating from '$COLLECTION_STATE_LOCATION' to storage units..." }

current.lastLevel = props.getProperty("lastLevel")?.let { DataCollection.fromDB(it) }
current.uuid = props.getProperty("uuid")?.let { UUID.fromString(it) }
setUUID(
props.getProperty("uuid")?.let { UUID.fromString(it) }
)

saveState()
deleteOldState()
deleteOldState()

logger.info { "Migration complete!" }
logger.info { "Migration complete!" }
} else {
logger.info {
"Removing '$COLLECTION_STATE_LOCATION' as UUID configured via system property or " +
"environmental variable."
}

deleteOldState()
}
}
}

Expand All @@ -85,12 +99,12 @@ public class DataCollector(public val level: DataCollection) : KordExKoinCompone
try {
lateinit var entity: Entity

val state = getState()
val lastUUID = getUUID()

when (level) {
is DataCollection.Minimal ->
entity = MinimalDataEntity(
id = state.uuid,
id = lastUUID,

devMode = settings.devMode,
kordExVersion = KORDEX_VERSION ?: "Unknown",
Expand All @@ -103,7 +117,7 @@ public class DataCollector(public val level: DataCollection) : KordExKoinCompone

is DataCollection.Standard ->
entity = StandardDataEntity(
id = state.uuid,
id = lastUUID,

devMode = settings.devMode,
kordExVersion = KORDEX_VERSION ?: "Unknown",
Expand All @@ -123,6 +137,34 @@ public class DataCollector(public val level: DataCollection) : KordExKoinCompone
?: arrayOf(),

pluginCount = settings.pluginBuilder.managerObj.plugins.size,

chatCommandCount = if (settings.chatCommandsBuilder.enabled) {
chatCommands.commands.size
} else {
0
},

messageCommandCount = applicationCommands
?.let {
it.messageCommands
.filterValues { it.guildId == null }
.size
},

slashCommandCount = applicationCommands
?.let {
it.slashCommands
.filterValues { it.guildId == null }
.size
},

userCommandCount = applicationCommands
?.let {
it.userCommands
.filterValues { it.guildId == null }
.size
},

jvmVersion = System.getProperty("java.version"),
kotlinVersion = KotlinVersion.CURRENT.toString(),
)
Expand All @@ -132,7 +174,7 @@ public class DataCollector(public val level: DataCollection) : KordExKoinCompone
val processor = hardware.processor

entity = ExtraDataEntity(
id = state.uuid,
id = lastUUID,

devMode = settings.devMode,
kordExVersion = KORDEX_VERSION ?: "Unknown",
Expand All @@ -153,6 +195,33 @@ public class DataCollector(public val level: DataCollection) : KordExKoinCompone

pluginCount = settings.pluginBuilder.managerObj.plugins.size,

chatCommandCount = if (settings.chatCommandsBuilder.enabled) {
chatCommands.commands.size
} else {
0
},

messageCommandCount = applicationCommands
?.let {
it.messageCommands
.filterValues { it.guildId == null }
.size
},

slashCommandCount = applicationCommands
?.let {
it.slashCommands
.filterValues { it.guildId == null }
.size
},

userCommandCount = applicationCommands
?.let {
it.userCommands
.filterValues { it.guildId == null }
.size
},

cpuCount = processor.physicalProcessorCount,
cpuGhz = processor.maxFreq.toFloat().div(1_000_000_000F), // GHz, not Hz

Expand All @@ -178,28 +247,31 @@ public class DataCollector(public val level: DataCollection) : KordExKoinCompone
}

else -> {
if (state.uuid != null) {
DataAPIClient.delete(state.uuid!!)

state.uuid = null
state.lastLevel = level

saveState()
if (lastUUID != null) {
logger.debug { "Deleting collected data - last UUID: $lastUUID" }

try {
DataAPIClient.delete(lastUUID)

setUUID(null)
} catch (e: ResponseException) {
logger.error {
"Failed to remove collected data '$lastUUID' from the server: $e\n" +
"\tThis will be re-attempted next time the bot starts. Please report this error " +
"to Kord Extensions via the community links here: https://kordex.dev"
}
}
}

return stop()
}
}

logger.debug { "Submitting collected data - level: ${level.readable}, last UUID: ${state.uuid}" }
logger.debug { "Submitting collected data - level: ${level.readable}, last UUID: $lastUUID" }

val response = DataAPIClient.submit(entity)
val lastUUID = state.uuid

state.uuid = response
state.lastLevel = level

saveState()
setUUID(response)

if (lastUUID == response) {
logger.debug { "Updated collected data successfully - UUID: $response" }
Expand All @@ -218,44 +290,44 @@ public class DataCollector(public val level: DataCollection) : KordExKoinCompone
internal suspend fun start() {
logger.info { "Staring data collector - level: ${level.readable}" }

if (applicationCommands == null) {
logger.info {
"Application command registry doesn't extend `DefaultApplicationCommandRegistry`, not collecting " +
"application command counts."
}
}

migrate()

val state = getState()
val lastUUID = getUUID()

if (state.lastLevel !is DataCollection.None && level is DataCollection.None) {
if (state.uuid != null) {
if (level is DataCollection.None) {
if (lastUUID != null) {
try {
DataAPIClient.delete(state.uuid!!)
logger.debug { "Deleting collected data - last UUID: $lastUUID" }

DataAPIClient.delete(lastUUID)
} catch (e: ResponseException) {
logger.error {
"Failed to remove collected data '${state.uuid}' from the server: $e\n" +
"Failed to remove collected data '$lastUUID' from the server: $e\n" +
"\tThis will be re-attempted next time the bot starts. Please report this error to Kord " +
"Extensions via the community links here: https://kordex.dev"
}

return stop()
}

logger.info { "Collected data '${state.uuid}' has been deleted from the server." }
setUUID(null)
}

state.uuid = null
state.lastLevel = level

saveState()

return stop()
}

state.lastLevel = level

saveState()

task = scheduler.schedule(
callback = ::collect,
delay = 30.minutes,
name = "DataCollector",
pollingSeconds = 600.seconds.inWholeSeconds,
pollingSeconds = 10.minutes.inWholeSeconds,
repeat = true,
startNow = true
)
Expand Down Expand Up @@ -307,6 +379,12 @@ public class DataCollector(public val level: DataCollection) : KordExKoinCompone
}

private suspend fun getState(): State {
if (DATA_COLLECTION_UUID != null) {
// Don't save anything if pre-configured.

return State(DATA_COLLECTION_UUID)
}

var current = storageUnit
.withUser(bot.kordRef.selfId)
.get()
Expand All @@ -323,16 +401,28 @@ public class DataCollector(public val level: DataCollection) : KordExKoinCompone
}

private suspend fun saveState() {
if (DATA_COLLECTION_UUID != null) {
// Don't save anything if pre-configured.

return
}

storageUnit
.withUser(bot.kordRef.selfId)
.save()
}

/** Get the stored "last" data collection level. **/
public suspend fun getLastLevel(): DataCollection? =
getState().lastLevel

/** Get the stored data collection UUID. **/
public suspend fun getUUID(): UUID? =
getState().uuid
DATA_COLLECTION_UUID
?: getState().uuid

/** Get the stored data collection UUID. **/
internal suspend fun setUUID(uuid: UUID?) {
val state = getState()

state.uuid = uuid

saveState()
}
}
3 changes: 0 additions & 3 deletions kord-extensions/src/main/kotlin/dev/kordex/core/datacollection/State.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,13 @@
package dev.kordex.core.datacollection

import dev.kordex.core.storage.Data
import dev.kordex.data.api.DataCollection
import dev.kordex.data.api.serializers.UUIDSerializer
import kotlinx.serialization.Serializable
import java.util.UUID

@Suppress("DataClassShouldBeImmutable")
@Serializable
public data class State(
var lastLevel: DataCollection? = DataCollection.Standard,

@Serializable(with = UUIDSerializer::class)
var uuid: UUID? = null,
) : Data
Loading

0 comments on commit 474a161

Please sign in to comment.