Skip to content

Commit

Permalink
Mbs 10991 utilization metric (#987)
Browse files Browse the repository at this point in the history
  • Loading branch information
RuslanMingaliev authored May 18, 2021
1 parent ee97b46 commit 59785be
Show file tree
Hide file tree
Showing 17 changed files with 303 additions and 193 deletions.
1 change: 1 addition & 0 deletions subprojects/test-runner/client/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ dependencies {
testImplementation(project(":gradle:test-project"))
testImplementation(testFixtures(project(":common:logger")))
testImplementation(testFixtures(project(":common:time")))
testImplementation(testFixtures(project(":test-runner:service")))
testImplementation(libs.kotlinReflect)
testImplementation(libs.mockitoKotlin)
testImplementation(libs.mockitoJUnitJupiter)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
package com.avito.runner.scheduler.metrics

import com.avito.android.Result

internal interface TestMetricsAggregator {

fun initialDelay(): Long?
fun initialDelay(): Result<Long>

fun endDelay(): Long?
fun endDelay(): Result<Long>

fun medianQueueTime(): Long?
fun medianQueueTime(): Result<Long>

fun medianInstallationTime(): Long?
fun medianInstallationTime(): Result<Long>

fun suiteTime(): Long?
fun suiteTime(): Result<Long>

fun totalTime(): Long

fun medianDeviceUtilization(): Long?
fun medianDeviceUtilization(): Result<Long>
}
Original file line number Diff line number Diff line change
@@ -1,66 +1,70 @@
package com.avito.runner.scheduler.metrics

import com.avito.android.Result
import com.avito.math.median
import com.avito.runner.scheduler.metrics.model.DeviceKey
import com.avito.runner.scheduler.metrics.model.DeviceTimestamps
import com.avito.runner.scheduler.metrics.model.TestTimestamps

internal class TestMetricsAggregatorImpl(
internal data class TestMetricsAggregatorImpl(
private val testSuiteStartedTime: Long,
private val testSuiteEndedTime: Long,
private val deviceTimestamps: Map<DeviceKey, DeviceTimestamps>
) : TestMetricsAggregator {

private val testTimestamps = deviceTimestamps.flatMap { it.value.testTimestamps.values }

private val firstTestStarted = testTimestamps
.mapNotNull { it.started }
private val firstTestStarted: Result<Long> = testTimestamps.filterIsInstance<TestTimestamps.Finished>()
.map { it.startTime }
.minOrNull()
.toResult { "Cannot calculate first started test time" }

private val lastTestEnded = testTimestamps
.mapNotNull { it.finished }
private val lastTestEnded: Result<Long> = testTimestamps.filterIsInstance<TestTimestamps.Finished>()
.map { it.finishTime }
.maxOrNull()
.toResult { "Cannot calculate last ended test time" }

private val queueTimes: List<Long> = testTimestamps
.mapNotNull { it.onDevice }
.map { it - testSuiteStartedTime }
private val queueTimes: List<Long> = testTimestamps.filterIsInstance<TestTimestamps.Finished>()
.map { it.onDevice - testSuiteStartedTime }

private val installationTimes: List<Long> = testTimestamps
.mapNotNull { it.installationTime }
private val installationTimes: List<Long> = testTimestamps.filterIsInstance<TestTimestamps.Finished>()
.map { it.installationTime }

override fun initialDelay(): Long? = firstTestStarted?.let { it - testSuiteStartedTime }
override fun initialDelay(): Result<Long> = firstTestStarted.map { it - testSuiteStartedTime }

override fun endDelay(): Long? = lastTestEnded?.let { testSuiteEndedTime - it }
override fun endDelay(): Result<Long> = lastTestEnded.map { testSuiteEndedTime - it }

override fun medianQueueTime(): Long? = queueTimes.aggregateOrNull { it.median() }
override fun medianQueueTime(): Result<Long> = queueTimes.aggregate(
{ it.median() },
{ "Cannot calculate median queue time" }
)

override fun medianInstallationTime(): Long? = installationTimes.aggregateOrNull { it.median() }
override fun medianInstallationTime(): Result<Long> = installationTimes.aggregate(
{ it.median() },
{ "Cannot calculate median installation time" }
)

override fun suiteTime(): Long? = if (lastTestEnded != null && firstTestStarted != null) {
lastTestEnded - firstTestStarted
} else {
null
}
override fun suiteTime(): Result<Long> = lastTestEnded.combine(firstTestStarted) { last, first -> last - first }

override fun totalTime() = testSuiteEndedTime - testSuiteStartedTime

override fun medianDeviceUtilization(): Long? =
deviceTimestamps.values
.mapNotNull { it.utilizationPercent }
.aggregateOrNull { it.median() }

/**
* return null if no data
*/
private fun List<Number>.aggregateOrNull(aggregateFunc: (List<Number>) -> Number): Long? {
return if (isNotEmpty()) {
val result = aggregateFunc.invoke(this).toLong()
if (result > 0) {
result
} else {
null
}
} else {
null
override fun medianDeviceUtilization(): Result<Long> =
deviceTimestamps.values.filterIsInstance<DeviceTimestamps.Finished>()
.map { it.utilizationPercent }
.aggregate({ it.median() }) { "Cannot calculate median device utilization" }

private inline fun Long?.toResult(lazyMessage: () -> String): Result<Long> = when (this) {
null -> Result.Failure(IllegalStateException(lazyMessage()))
else -> Result.Success(this)
}

private inline fun List<Number>.aggregate(
aggregateFunc: (List<Number>) -> Number,
lazyMessage: () -> String
): Result<Long> {
return when {
this.isEmpty() -> Result.Failure(IllegalStateException(lazyMessage()))
else -> Result.Success(aggregateFunc(this).toLong())
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import com.avito.runner.scheduler.metrics.model.DeviceKey
import com.avito.runner.scheduler.metrics.model.DeviceTimestamps
import com.avito.runner.scheduler.metrics.model.TestKey
import com.avito.runner.scheduler.metrics.model.TestTimestamps
import com.avito.runner.scheduler.metrics.model.finish
import com.avito.runner.scheduler.metrics.model.start
import com.avito.runner.service.model.DeviceTestCaseRun
import com.avito.runner.service.model.intention.Intention
import com.avito.runner.service.model.intention.State
Expand All @@ -31,10 +33,9 @@ internal class TestMetricsListenerImpl(
}

override suspend fun onDeviceCreated(device: Device, state: State) {
deviceTimestamps[device.key()] = DeviceTimestamps(
deviceTimestamps[device.key()] = DeviceTimestamps.Started(
created = timeProvider.nowInMillis(),
testTimestamps = mutableMapOf(),
finished = 0
)
}

Expand All @@ -44,20 +45,18 @@ internal class TestMetricsListenerImpl(
logger.warn("Fail to set timestamp value, previous required values not found, this shouldn't happen")
null
} else {
oldValue.testTimestamps[intention.testKey()] = TestTimestamps(
onDevice = timeProvider.nowInMillis(),
started = null,
finished = null
)
oldValue.testTimestamps[intention.testKey()] = TestTimestamps.NotStarted(timeProvider.nowInMillis())
oldValue
}
}
}

override suspend fun onStatePrepared(device: Device, state: State) {
// empty
}

override suspend fun onApplicationInstalled(device: Device, installation: DeviceInstallation) {
// empty
}

override suspend fun onTestStarted(device: Device, intention: Intention) {
Expand All @@ -74,7 +73,7 @@ internal class TestMetricsListenerImpl(
)
null
} else {
testTimestamps.copy(started = timeProvider.nowInMillis())
testTimestamps.start(timeProvider.nowInMillis())
}
}
oldValue
Expand All @@ -96,7 +95,7 @@ internal class TestMetricsListenerImpl(
)
null
} else {
testTimestamps.copy(finished = timeProvider.nowInMillis())
testTimestamps.finish(timeProvider.nowInMillis())
}
}
oldValue
Expand All @@ -105,6 +104,7 @@ internal class TestMetricsListenerImpl(
}

override suspend fun onIntentionFail(device: Device, intention: Intention, reason: Throwable) {
// empty
}

override suspend fun onDeviceDied(device: Device, message: String, reason: Throwable) {
Expand All @@ -117,7 +117,7 @@ internal class TestMetricsListenerImpl(
logger.warn("Fail to set timestamp value, previous required values not found, this shouldn't happen")
null
} else {
oldValue.copy(finished = timeProvider.nowInMillis())
oldValue.finish(finished = timeProvider.nowInMillis())
}
}
}
Expand All @@ -126,31 +126,36 @@ internal class TestMetricsListenerImpl(
val aggregator: TestMetricsAggregator = createTestMetricsAggregator()

with(testMetricsSender) {
aggregator.initialDelay()
?.let { sendInitialDelay(it) }
?: logger.warn("Not sending initial delay, no data")

aggregator.medianQueueTime()
?.let { sendMedianQueueTime(it) }
?: logger.warn("Not sending median test queue time, no data")

aggregator.medianInstallationTime()
?.let { sendMedianInstallationTime(it) }
?: logger.warn("Not sending median test start time, no data")

aggregator.endDelay()
?.let { sendEndDelay(it) }
?: logger.warn("Not sending end delay, no data")

aggregator.suiteTime()
?.let { sendSuiteTime(it) }
?: logger.warn("Not sending suite time, no data")
aggregator.initialDelay().fold(
{ sendInitialDelay(it) },
{ logger.warn("Not sending initial delay, no data") }
)

aggregator.medianQueueTime().fold(
{ sendMedianQueueTime(it) },
{ logger.warn("Not sending median test queue time, no data") }
)
aggregator.medianInstallationTime().fold(
{ sendMedianInstallationTime(it) },
{ logger.warn("Not sending median test start time, no data") }
)

aggregator.endDelay().fold(
{ sendEndDelay(it) },
{ logger.warn("Not sending end delay, no data") }
)

aggregator.suiteTime().fold(
{ sendSuiteTime(it) },
{ logger.warn("Not sending suite time, no data") }
)

sendTotalTime(aggregator.totalTime())

aggregator.medianDeviceUtilization()
?.let { sendMedianDeviceUtilization(it.toInt()) }
?: logger.warn("Not sending median device relative wasted time, no data")
aggregator.medianDeviceUtilization().fold(
{ sendMedianDeviceUtilization(it.toInt()) },
{ logger.warn("Not sending median device relative wasted time, no data") }
)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,6 @@ internal class TestMetricsSender(
}

fun sendMedianDeviceUtilization(percent: Int) {
statsDSender.send(GaugeLongMetric(prefix.append("device-utilization.median"), percent.toLong()))
statsDSender.send(GaugeLongMetric(prefix.append("device-utilization", "median"), percent.toLong()))
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,35 @@ package com.avito.runner.scheduler.metrics.model

import com.avito.math.percentOf

internal data class DeviceTimestamps(
val created: Long? = null,
val testTimestamps: MutableMap<TestKey, TestTimestamps>,
val finished: Long? = null
internal sealed class DeviceTimestamps(
open val created: Long,
open val testTimestamps: MutableMap<TestKey, TestTimestamps>,
) {

private val totalTime: Long?
get() = if (finished != null && created != null) {
finished - created
} else {
null
}

private val effectiveWorkTime
get() = testTimestamps.values.mapNotNull { it.effectiveWorkTime }.sum()

val utilizationPercent: Int?
get() {
val localTotalTime = totalTime
return if (localTotalTime != null) {
effectiveWorkTime.percentOf(localTotalTime).toInt()
} else {
null // in case if device died and finish time not logged
}
}
data class Started(
override val created: Long,
override val testTimestamps: MutableMap<TestKey, TestTimestamps>,
) : DeviceTimestamps(created, testTimestamps)

data class Finished(
override val created: Long,
override val testTimestamps: MutableMap<TestKey, TestTimestamps>,
val finished: Long
) : DeviceTimestamps(created, testTimestamps) {

private val totalTime = finished - created

private val effectiveWorkTime = testTimestamps.values.filterIsInstance<TestTimestamps.Finished>()
.map { it.effectiveWorkTime }
.sum()

val utilizationPercent: Int = effectiveWorkTime.percentOf(totalTime).toInt()
}

companion object
}

internal fun DeviceTimestamps.finish(finished: Long): DeviceTimestamps = when (this) {
is DeviceTimestamps.Started -> DeviceTimestamps.Finished(this.created, this.testTimestamps, finished)
is DeviceTimestamps.Finished -> error("$this is in its finished state already")
}
Original file line number Diff line number Diff line change
@@ -1,27 +1,25 @@
package com.avito.runner.scheduler.metrics.model

internal data class TestTimestamps(
val onDevice: Long? = null,
val started: Long? = null,
val finished: Long? = null
) {
val effectiveWorkTime: Long? = if (finished != null && onDevice != null) {
finished - onDevice
} else {
null
}
internal sealed class TestTimestamps {

val installationTime: Long? = if (started != null && onDevice != null) {
started - onDevice
} else {
null
}
data class NotStarted(val onDevice: Long) : TestTimestamps()

data class Started(val onDevice: Long, val startTime: Long) : TestTimestamps()

val executionTime: Long? = if (finished != null && started != null) {
finished - started
} else {
null
data class Finished(val onDevice: Long, val startTime: Long, val finishTime: Long) : TestTimestamps() {
val effectiveWorkTime = finishTime - onDevice
val installationTime = startTime - onDevice
}
}

internal fun TestTimestamps.start(currentTimeMillis: Long): TestTimestamps = when (this) {
is TestTimestamps.NotStarted -> TestTimestamps.Started(this.onDevice, currentTimeMillis)
is TestTimestamps.Started -> error("Can't start already started $this")
is TestTimestamps.Finished -> error("Can't start already finished $this")
}

companion object
internal fun TestTimestamps.finish(currentTimeMillis: Long): TestTimestamps = when (this) {
is TestTimestamps.NotStarted -> error("Can't finish not started $this")
is TestTimestamps.Started -> TestTimestamps.Finished(this.onDevice, this.startTime, currentTimeMillis)
is TestTimestamps.Finished -> error("Can't finish already finished $this")
}
Loading

0 comments on commit 59785be

Please sign in to comment.