From ca85be66f339b08aee50319a7245810f760f6933 Mon Sep 17 00:00:00 2001 From: Michel Kraemer Date: Tue, 22 Oct 2024 10:42:26 +0200 Subject: [PATCH] Add capability matcher plugin type --- src/main/kotlin/agent/RemoteAgentRegistry.kt | 6 +- src/main/kotlin/cloud/SetupSelector.kt | 14 +-- src/main/kotlin/db/PluginRegistry.kt | 8 ++ src/main/kotlin/db/PluginRegistryFactory.kt | 4 + src/main/kotlin/helper/CapabilitiesMatcher.kt | 2 +- .../plugins/CapabilitiesMatcherPlugin.kt | 108 ++++++++++++++++++ src/main/kotlin/model/plugins/Plugin.kt | 1 + .../model/plugins/SetupAdapterPlugin.kt | 4 +- src/test/kotlin/db/PluginRegistryTest.kt | 46 ++++++++ .../resources/db/dummyCapabilityMatcher.kts | 9 ++ .../resources/db/dummyCapabilityMatcher.yaml | 5 + 11 files changed, 194 insertions(+), 13 deletions(-) create mode 100644 src/main/kotlin/model/plugins/CapabilitiesMatcherPlugin.kt create mode 100644 src/test/resources/db/dummyCapabilityMatcher.kts create mode 100644 src/test/resources/db/dummyCapabilityMatcher.yaml diff --git a/src/main/kotlin/agent/RemoteAgentRegistry.kt b/src/main/kotlin/agent/RemoteAgentRegistry.kt index 054ed772..57da71ff 100644 --- a/src/main/kotlin/agent/RemoteAgentRegistry.kt +++ b/src/main/kotlin/agent/RemoteAgentRegistry.kt @@ -5,7 +5,7 @@ import AddressConstants.REMOTE_AGENT_ADDED import AddressConstants.REMOTE_AGENT_ADDRESS_PREFIX import AddressConstants.REMOTE_AGENT_LEFT import agent.AgentRegistry.SelectCandidatesParam -import helper.CapabilitiesMatcher +import helper.CapabilityMatcher import helper.JsonUtils import helper.debounce import helper.hazelcast.ClusterMap @@ -116,7 +116,7 @@ class RemoteAgentRegistry(private val vertx: Vertx) : AgentRegistry, CoroutineSc * Matches required capabilities of process chains against provided * capabilities of agents */ - private val capabilitiesMatcher = CapabilitiesMatcher() + private val capabilityMatcher = CapabilityMatcher() init { // create shared maps @@ -281,7 +281,7 @@ class RemoteAgentRegistry(private val vertx: Vertx) : AgentRegistry, CoroutineSc // agent's lastSequence would definitely be higher continue } - if (capabilitiesMatcher.matches(ps.requiredCapabilities, supportedCapabilities)) { + if (capabilityMatcher.matches(ps.requiredCapabilities, supportedCapabilities)) { // the agent supports at least one required capabilities set shouldInquire = true break diff --git a/src/main/kotlin/cloud/SetupSelector.kt b/src/main/kotlin/cloud/SetupSelector.kt index d40fb682..6fe45808 100644 --- a/src/main/kotlin/cloud/SetupSelector.kt +++ b/src/main/kotlin/cloud/SetupSelector.kt @@ -1,7 +1,7 @@ package cloud import db.VMRegistry -import helper.CapabilitiesMatcher +import helper.CapabilityMatcher import model.cloud.PoolAgentParams import model.setup.Setup import org.slf4j.LoggerFactory @@ -22,7 +22,7 @@ class SetupSelector(private val vmRegistry: VMRegistry, /** * Matches required capabilities with those provided by setups */ - private val capabilitiesMatcher = CapabilitiesMatcher() + private val capabilityMatcher = CapabilityMatcher() /** * Iterate through [nVMsPerSetup] and [nCreatedPerSetup] and count how many @@ -35,7 +35,7 @@ class SetupSelector(private val vmRegistry: VMRegistry, .groupBy { it.key }.mapValues { e -> e.value.sumOf { v -> v.value } } return merged.map { (setup, n) -> - if (capabilitiesMatcher.matches(capabilities, setup.providedCapabilities)) { + if (capabilityMatcher.matches(capabilities, setup.providedCapabilities)) { n } else { 0 @@ -68,7 +68,7 @@ class SetupSelector(private val vmRegistry: VMRegistry, setups: List): List { // select candidate setups that satisfy the given required capabilities val matchingSetups = setups.filter { s -> - capabilitiesMatcher.matches(requiredCapabilities, s.providedCapabilities) + capabilityMatcher.matches(requiredCapabilities, s.providedCapabilities) } if (matchingSetups.isEmpty()) { return emptyList() @@ -99,7 +99,7 @@ class SetupSelector(private val vmRegistry: VMRegistry, // check if we already have enough VMs that provide similar capabilities for (params in poolAgentParams) { - if (params.max != null && capabilitiesMatcher.matches(params.capabilities, + if (params.max != null && capabilityMatcher.matches(params.capabilities, setup.providedCapabilities)) { // Creating a new VM with [setup] would add an agent with the given // provided capabilities. Check if this would exceed the maximum @@ -165,7 +165,7 @@ class SetupSelector(private val vmRegistry: VMRegistry, val papi = pap.iterator() while (papi.hasNext()) { val p = papi.next() - if (capabilitiesMatcher.matches(p.capabilities, setup.providedCapabilities)) { + if (capabilityMatcher.matches(p.capabilities, setup.providedCapabilities)) { // we found parameters our setup satisfies if (p.min > minimum) { minimum = min(p.min, maximum) @@ -195,7 +195,7 @@ class SetupSelector(private val vmRegistry: VMRegistry, val i = result.iterator() while (i.hasNext()) { val setup = i.next() - if (capabilitiesMatcher.matches(p.capabilities, setup.providedCapabilities)) { + if (capabilityMatcher.matches(p.capabilities, setup.providedCapabilities)) { if (count == p.max) { i.remove() } else { diff --git a/src/main/kotlin/db/PluginRegistry.kt b/src/main/kotlin/db/PluginRegistry.kt index 32fe4b29..befb9fef 100644 --- a/src/main/kotlin/db/PluginRegistry.kt +++ b/src/main/kotlin/db/PluginRegistry.kt @@ -1,5 +1,6 @@ package db +import model.plugins.CapabilityMatcherPlugin import model.plugins.InitializerPlugin import model.plugins.InputAdapterPlugin import model.plugins.OutputAdapterPlugin @@ -15,6 +16,8 @@ import model.plugins.SetupAdapterPlugin * @author Michel Kraemer */ class PluginRegistry(private val compiledPlugins: List) { + private val capabilityMatchers = compiledPlugins + .filterIsInstance() private val initializers = compiledPlugins.filterIsInstance() .toResolved() private val inputAdapters = compiledPlugins.filterIsInstance() @@ -40,6 +43,11 @@ class PluginRegistry(private val compiledPlugins: List) { */ fun getAllPlugins() = compiledPlugins + /** + * Get all capability matchers + */ + fun getCapabilityMatchers() = capabilityMatchers + /** * Get all initializers */ diff --git a/src/main/kotlin/db/PluginRegistryFactory.kt b/src/main/kotlin/db/PluginRegistryFactory.kt index 85622b50..d22beac8 100644 --- a/src/main/kotlin/db/PluginRegistryFactory.kt +++ b/src/main/kotlin/db/PluginRegistryFactory.kt @@ -7,6 +7,7 @@ import io.vertx.core.Vertx import io.vertx.core.json.JsonArray import io.vertx.core.json.JsonObject import io.vertx.kotlin.coroutines.coAwait +import model.plugins.CapabilityMatcherPlugin import model.plugins.InitializerPlugin import model.plugins.InputAdapterPlugin import model.plugins.OutputAdapterPlugin @@ -16,6 +17,7 @@ import model.plugins.ProcessChainConsistencyCheckerPlugin import model.plugins.ProgressEstimatorPlugin import model.plugins.RuntimePlugin import model.plugins.SetupAdapterPlugin +import model.plugins.capabilityMatcherPluginTemplate import model.plugins.initializerPluginTemplate import model.plugins.inputAdapterPluginTemplate import model.plugins.outputAdapterPluginTemplate @@ -237,6 +239,8 @@ object PluginRegistryFactory { @Suppress("UNCHECKED_CAST") return when (plugin) { + is CapabilityMatcherPlugin -> plugin.copy(compiledFunction = wrapPluginFunction( + f as KFunction, ::capabilityMatcherPluginTemplate.parameters)) is InitializerPlugin -> plugin.copy(compiledFunction = wrapPluginFunction( f as KFunction, ::initializerPluginTemplate.parameters)) is InputAdapterPlugin -> plugin.copy(compiledFunction = wrapPluginFunction( diff --git a/src/main/kotlin/helper/CapabilitiesMatcher.kt b/src/main/kotlin/helper/CapabilitiesMatcher.kt index 837a408a..23f85cfc 100644 --- a/src/main/kotlin/helper/CapabilitiesMatcher.kt +++ b/src/main/kotlin/helper/CapabilitiesMatcher.kt @@ -7,7 +7,7 @@ package helper * [CapabilityMatcherPlugin]s doing custom comparisons. * @author Michel Kraemer */ -class CapabilitiesMatcher { +class CapabilityMatcher { /** * Matches a collection of [requiredCapabilities] with a collection of * [providedCapabilities]. Returns `true` if the collection of provided diff --git a/src/main/kotlin/model/plugins/CapabilitiesMatcherPlugin.kt b/src/main/kotlin/model/plugins/CapabilitiesMatcherPlugin.kt new file mode 100644 index 00000000..15c41147 --- /dev/null +++ b/src/main/kotlin/model/plugins/CapabilitiesMatcherPlugin.kt @@ -0,0 +1,108 @@ +package model.plugins + +import com.fasterxml.jackson.annotation.JsonIgnore +import io.vertx.core.Vertx +import kotlin.reflect.KFunction +import kotlin.reflect.full.callSuspend + +/** + * Parameters that will be passed to a [CapabilityMatcherPlugin] + */ +interface CapabilityMatcherParams { + /** + * The required capability that should be matched with the collection of + * [providedCapabilities]. In the simplest case, the plugin may just check + * if the collection contains this string, but more complex comparisons + * can be implemented. + */ + val subject: String + + /** + * The collection of provided capabilities to which the required capability + * denoted with [subject] should be matched. + */ + val providedCapabilities: Collection + + /** + * The collection of all required capabilities (including [subject]) for + * reference or if complex decisions need to be made (e.g. [subject] `"A"` + * only matches [providedCapabilities] if [allRequiredCapabilities] does not + * contain string `"B"`). + */ + val allRequiredCapabilities: Collection +} + +/** + * A simple implementation of [CapabilityMatcherParams] + */ +data class SimpleCapabilityMatcherParams( + override val subject: String, + override val providedCapabilities: Collection, + override val allRequiredCapabilities: Collection +) : CapabilityMatcherParams + +/** + * A capability matcher plugin is a function that can change how Steep decides + * whether a given collection of provided capabilities satisfy a required + * capability. The function has the following signature: + * + * suspend fun myCapabilityMatcher( + * params: model.plugins.CapabilityMatcherParams, + * vertx: io.vertx.core.Vertx): Int + * + * The function will be called with a set of [CapabilityMatcherParams] and + * the Vert.x instance. If required, the function can be a suspend function. + * + * The parameters contain a [CapabilityMatcherParams.subject] (representing + * a single required capability) and a collection of + * [CapabilityMatcherParams.providedCapabilities]. + * + * For reference or if more complex decisions need to be made, the parameters + * also contain the collection of all required capabilities (including the + * subject). + * + * The function can cast a vote on whether the provided capabilities satisfy + * the required capability denoted by [CapabilityMatcherParams.subject]. For + * this, it returns an integer value. Positive values mean that the provided + * capabilities match the subject. The higher the value, the more certain the + * function is about this. A value of [Int.MAX_VALUE] means that the + * capabilities definitely match (i.e. the plugin is absolutely certain). No + * other plugin will be called in this case. + * + * Negative values mean that the provided capabilities *do not* match the + * subject. The lower the value, the more certain the function is about this. + * A value of [Int.MIN_VALUE] means that the plugin is absolutely certain that + * capabilities *do not* match. No other plugin will be called in this case. + * + * A value of 0 (zero) means that the plugin is unable to tell if the provided + * capabilities match the subject. The decision should be made based on other + * criteria (i.e. by calling other plugins or by simply comparing strings as + * a fallback). + */ +data class CapabilityMatcherPlugin( + override val name: String, + override val scriptFile: String, + override val version: String? = null, + + /** + * The compiled plugin + */ + @JsonIgnore + override val compiledFunction: KFunction = throwPluginNeedsCompile() +) : Plugin + +@Suppress("UNUSED_PARAMETER") +internal fun capabilityMatcherPluginTemplate(params: CapabilityMatcherParams, + vertx: Vertx): Int { + throw NotImplementedError("This is just a template specifying the " + + "function signature of a setup adapter plugin") +} + +suspend fun CapabilityMatcherPlugin.call(params: CapabilityMatcherParams, + vertx: Vertx): Int { + return if (this.compiledFunction.isSuspend) { + this.compiledFunction.callSuspend(params, vertx) + } else { + this.compiledFunction.call(params, vertx) + } +} diff --git a/src/main/kotlin/model/plugins/Plugin.kt b/src/main/kotlin/model/plugins/Plugin.kt index 0d8d02d6..c0969baa 100644 --- a/src/main/kotlin/model/plugins/Plugin.kt +++ b/src/main/kotlin/model/plugins/Plugin.kt @@ -18,6 +18,7 @@ import kotlin.reflect.jvm.javaType include = JsonTypeInfo.As.PROPERTY, property = "type") @JsonSubTypes( + JsonSubTypes.Type(value = CapabilityMatcherPlugin::class, name = "capabilityMatcher"), JsonSubTypes.Type(value = InitializerPlugin::class, name = "initializer"), JsonSubTypes.Type(value = InputAdapterPlugin::class, name = "inputAdapter"), JsonSubTypes.Type(value = OutputAdapterPlugin::class, name = "outputAdapter"), diff --git a/src/main/kotlin/model/plugins/SetupAdapterPlugin.kt b/src/main/kotlin/model/plugins/SetupAdapterPlugin.kt index 96a5276a..fa28eaf7 100644 --- a/src/main/kotlin/model/plugins/SetupAdapterPlugin.kt +++ b/src/main/kotlin/model/plugins/SetupAdapterPlugin.kt @@ -13,12 +13,12 @@ import kotlin.reflect.full.callSuspend * * suspend fun mySetupAdapter(setup: model.setup.Setup, * requiredCapabilities: Collection, - * vertx: io.vertx.core.Vertx): model.setup.Setup, + * vertx: io.vertx.core.Vertx): model.setup.Setup * * The function will be called with a setup to modify and a collection of * required capabilities the modified setup should meet. The function should * return a new setup instance or the original one if no modifications were - * necessary. + * necessary. If required, the function can be a suspend function. */ data class SetupAdapterPlugin( override val name: String, diff --git a/src/test/kotlin/db/PluginRegistryTest.kt b/src/test/kotlin/db/PluginRegistryTest.kt index 8657bb02..0a673399 100644 --- a/src/test/kotlin/db/PluginRegistryTest.kt +++ b/src/test/kotlin/db/PluginRegistryTest.kt @@ -14,6 +14,7 @@ import io.vertx.kotlin.core.json.jsonObjectOf import io.vertx.kotlin.coroutines.dispatcher import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.launch +import model.plugins.CapabilityMatcherPlugin import model.plugins.InitializerPlugin import model.plugins.InputAdapterPlugin import model.plugins.OutputAdapterPlugin @@ -21,6 +22,7 @@ import model.plugins.ProcessChainAdapterPlugin import model.plugins.ProgressEstimatorPlugin import model.plugins.RuntimePlugin import model.plugins.SetupAdapterPlugin +import model.plugins.SimpleCapabilityMatcherParams import model.plugins.call import model.processchain.Argument import model.processchain.ArgumentVariable @@ -86,6 +88,50 @@ class PluginRegistryTest { } } + /** + * Test if [PluginRegistry.getCapabilityMatchers] works correctly + */ + @Test + fun getCapabilityMatchers() { + val cm1 = CapabilityMatcherPlugin("a", "file.kts") + val cm2 = CapabilityMatcherPlugin("b", "file2.kts") + val cm3 = CapabilityMatcherPlugin("c", "file3.kts") + val expected = listOf(cm1, cm2, cm3) + val pr = PluginRegistry(expected) + assertThat(pr.getCapabilityMatchers()).isEqualTo(expected) + assertThat(pr.getCapabilityMatchers()).isNotSameAs(expected) + } + + /** + * Test if a simple capability matcher can be compiled and executed + */ + @Test + fun compileDummyCapabilityMatcher(vertx: Vertx, ctx: VertxTestContext) { + CoroutineScope(vertx.dispatcher()).launch { + val config = jsonObjectOf( + ConfigConstants.PLUGINS to "src/**/db/dummyCapabilityMatcher.yaml" + ) + PluginRegistryFactory.initialize(vertx, config) + + val pr = PluginRegistryFactory.create() + val matchers = pr.getCapabilityMatchers() + ctx.coVerify { + assertThat(matchers).hasSize(1) + val matcher = matchers[0] + + val matches = matcher.call(SimpleCapabilityMatcherParams( + "elvis", listOf("max", "elvis", "sean"), listOf("elvis")), vertx) + assertThat(matches).isEqualTo(1) + + val doesNotMatch = matcher.call(SimpleCapabilityMatcherParams( + "elvis", listOf("max", "sean"), listOf("elvis")), vertx) + assertThat(doesNotMatch).isEqualTo(-1) + } + + ctx.completeNow() + } + } + /** * Test if [PluginRegistry.getInitializers] works correctly */ diff --git a/src/test/resources/db/dummyCapabilityMatcher.kts b/src/test/resources/db/dummyCapabilityMatcher.kts new file mode 100644 index 00000000..4af88a86 --- /dev/null +++ b/src/test/resources/db/dummyCapabilityMatcher.kts @@ -0,0 +1,9 @@ +import model.plugins.CapabilityMatcherParams + +fun dummyCapabilityMatcher(params: CapabilityMatcherParams): Int { + return if (params.providedCapabilities.contains(params.subject)) { + 1 + } else { + -1 + } +} diff --git a/src/test/resources/db/dummyCapabilityMatcher.yaml b/src/test/resources/db/dummyCapabilityMatcher.yaml new file mode 100644 index 00000000..c8d3cb4c --- /dev/null +++ b/src/test/resources/db/dummyCapabilityMatcher.yaml @@ -0,0 +1,5 @@ +# A capability matcher that returns 1 if `subject` is contained in the +# collection of provided capabilities and -1 if it's not. +- name: dummyCapabilityMatcher + type: capabilityMatcher + scriptFile: db/dummyCapabilityMatcher.kts