Skip to content

Commit

Permalink
Remove unnecessary hard-coded delay from K8s panel match test. (#1515)
Browse files Browse the repository at this point in the history 
This also cleans up some warnings.
  • Loading branch information
@SanjayVas
SanjayVas authored Mar 7, 2024
1 parent 0b99160 commit e62feab
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 55 deletions.
1 change: 1 addition & 0 deletions src/test/kotlin/org/wfanet/panelmatch/integration/k8s/AbstractPanelMatchCorrectnessTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ abstract class AbstractPanelMatchCorrectnessTest(private val localSystem: PanelM
)
}

@JvmStatic
protected suspend fun KubernetesClient.waitUntilDeploymentReady(name: String): V1Deployment {
logger.info { "Waiting for Deployment $name to be ready..." }
return waitUntilDeploymentReady(name, timeout = READY_TIMEOUT).also {
Expand Down
100 changes: 45 additions & 55 deletions ...est/kotlin/org/wfanet/panelmatch/integration/k8s/EmptyClusterPanelMatchCorrectnessTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,21 @@

package org.wfanet.panelmatch.integration.k8s

import com.google.common.util.concurrent.Futures.withTimeout
import com.google.protobuf.Any
import com.google.protobuf.kotlin.toByteString
import io.grpc.ManagedChannel
import io.kubernetes.client.common.KubernetesObject
import io.kubernetes.client.openapi.Configuration
import io.kubernetes.client.openapi.models.V1Deployment
import io.kubernetes.client.util.ClientBuilder
import java.io.File
import java.net.InetSocketAddress
import java.nio.charset.StandardCharsets
import java.nio.file.Paths
import java.security.Security
import java.time.Duration
import java.util.*
import java.util.UUID
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.time.withTimeout
import kotlinx.coroutines.withContext
Expand All @@ -48,7 +47,6 @@ import org.wfanet.measurement.api.v2alpha.ExchangesGrpcKt
import org.wfanet.measurement.api.v2alpha.ModelProviderKey
import org.wfanet.measurement.common.crypto.jceProvider
import org.wfanet.measurement.common.grpc.buildMutualTlsChannel
import org.wfanet.measurement.common.grpc.withContext
import org.wfanet.measurement.common.grpc.withDefaultDeadline
import org.wfanet.measurement.common.identity.apiIdToExternalId
import org.wfanet.measurement.common.k8s.KubernetesClient
Expand Down Expand Up @@ -121,15 +119,15 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest(
private val channels = mutableListOf<ManagedChannel>()
private val portForwarders = mutableListOf<PortForwarder>()

lateinit var dataProviderPrivateStorageDetails: StorageDetails
lateinit var dataProviderSharedStorageDetails: StorageDetails
lateinit var modelProviderPrivateStorageDetails: StorageDetails
lateinit var modelProviderSharedStorageDetails: StorageDetails
lateinit var dpForwardedStorage: StorageClient
lateinit var mpForwardedStorage: StorageClient
lateinit var publicChannel: ManagedChannel
lateinit var dataProviderDefaults: DaemonStorageClientDefaults
lateinit var modelProviderDefaults: DaemonStorageClientDefaults
private lateinit var dataProviderPrivateStorageDetails: StorageDetails
private lateinit var dataProviderSharedStorageDetails: StorageDetails
private lateinit var modelProviderPrivateStorageDetails: StorageDetails
private lateinit var modelProviderSharedStorageDetails: StorageDetails
private lateinit var dpForwardedStorage: StorageClient
private lateinit var mpForwardedStorage: StorageClient
private lateinit var publicChannel: ManagedChannel
private lateinit var dataProviderDefaults: DaemonStorageClientDefaults
private lateinit var modelProviderDefaults: DaemonStorageClientDefaults

override val runId: String by runId

Expand All @@ -142,8 +140,7 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest(
override fun evaluate() {
runBlocking {
withTimeout(Duration.ofMinutes(5)) {
var entitiesData = populateCluster()
_testHarness = createTestHarness(entitiesData)
_testHarness = createTestHarness(populateCluster())
}
}
base.evaluate()
Expand All @@ -162,15 +159,12 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest(
k8sClient.waitForServiceAccount("default", timeout = READY_TIMEOUT)

loadKingdomForPanelMatch()
val entitiesData = runResourceSetup(createEntityContent("edp1"), createEntityContent("mp1"))

logger.info { "Wait 20s for deployment to be ready" }
delay(20000)

return entitiesData
val dataProviderEntityContent = withContext(Dispatchers.IO) { createEntityContent("edp1") }
val modelProviderEntityContent = withContext(Dispatchers.IO) { createEntityContent("mp1") }
return runResourceSetup(dataProviderEntityContent, modelProviderEntityContent)
}

protected suspend fun runResourceSetup(
private suspend fun runResourceSetup(
dataProviderContent: EntityContent,
modelProviderContent: EntityContent,
): EntitiesData {
Expand Down Expand Up @@ -314,22 +308,27 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest(
}

private suspend fun loadKingdomForPanelMatch() {
withContext(Dispatchers.IO) {
val outputDir = tempDir.newFolder("kingdom-for-panelmatch-setup")
extractTar(
getRuntimePath(LOCAL_K8S_PATH.resolve("kingdom_for_panelmatch_setup.tar")).toFile(),
outputDir,
)
val config: File = outputDir.resolve("config.yaml")
kustomize(
outputDir
.toPath()
.resolve(LOCAL_K8S_PATH)
.resolve("kingdom_for_panelmatch_setup")
.toFile(),
config,
)
kubectlApply(config, k8sClient)
val appliedObjects: List<KubernetesObject> =
withContext(Dispatchers.IO) {
val outputDir = tempDir.newFolder("kingdom-for-panelmatch-setup")
extractTar(
getRuntimePath(LOCAL_K8S_PATH.resolve("kingdom_for_panelmatch_setup.tar")).toFile(),
outputDir,
)
val config: File = outputDir.resolve("config.yaml")
kustomize(
outputDir
.toPath()
.resolve(LOCAL_K8S_PATH)
.resolve("kingdom_for_panelmatch_setup")
.toFile(),
config,
)
kubectlApply(config, k8sClient)
}

appliedObjects.filterIsInstance<V1Deployment>().forEach {
k8sClient.waitUntilDeploymentReady(checkNotNull(it.metadata?.name))
}
}

Expand All @@ -341,10 +340,7 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest(
withContext(Dispatchers.IO) {
val outputDir = tempDir.newFolder("edp_daemon")
extractTar(
AbstractPanelMatchCorrectnessTest.getRuntimePath(
LOCAL_K8S_PANELMATCH_PATH.resolve("edp_daemon.tar")
)
.toFile(),
getRuntimePath(LOCAL_K8S_PANELMATCH_PATH.resolve("edp_daemon.tar")).toFile(),
outputDir,
)

Expand Down Expand Up @@ -373,10 +369,7 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest(
withContext(Dispatchers.IO) {
val outputDir = tempDir.newFolder("mp_daemon")
extractTar(
AbstractPanelMatchCorrectnessTest.getRuntimePath(
LOCAL_K8S_PANELMATCH_PATH.resolve("mp_daemon.tar")
)
.toFile(),
getRuntimePath(LOCAL_K8S_PANELMATCH_PATH.resolve("mp_daemon.tar")).toFile(),
outputDir,
)

Expand Down Expand Up @@ -470,8 +463,8 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest(
private val IMAGE_PANEL_MATCH_PUSHER_PATH =
Paths.get("src", "main", "docker", "panel_exchange_client", "push_all_images.bash")

private val API_VERSION = "v2alpha"
private val SCHEDULE = "@daily"
private const val API_VERSION = "v2alpha"
private const val SCHEDULE = "@daily"

private val tempDir = TemporaryFolder()

Expand All @@ -491,26 +484,23 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest(
}

@Blocking
protected fun kubectlApply(
config: String,
k8sClient: KubernetesClient,
): List<KubernetesObject> {
private fun kubectlApply(config: String, k8sClient: KubernetesClient): List<KubernetesObject> {
return k8sClient
.kubectlApply(config)
.onEach { logger.info { "Applied ${it.kind} ${it.metadata.name}" } }
.toList()
}

@Blocking
protected fun kubectlApply(config: File, k8sClient: KubernetesClient): List<KubernetesObject> {
private fun kubectlApply(config: File, k8sClient: KubernetesClient): List<KubernetesObject> {
return k8sClient
.kubectlApply(config)
.onEach { logger.info { "Applied ${it.kind} ${it.metadata.name}" } }
.toList()
}

@Blocking
protected fun kustomize(kustomizationDir: File, output: File) {
private fun kustomize(kustomizationDir: File, output: File) {
Processes.runCommand(
"kubectl",
"kustomize",
Expand All @@ -520,7 +510,7 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest(
)
}

protected fun extractTar(archive: File, outputDirectory: File) {
private fun extractTar(archive: File, outputDirectory: File) {
Processes.runCommand("tar", "-xf", archive.toString(), "-C", outputDirectory.toString())
}
}
Expand Down

0 comments on commit e62feab

Please sign in to comment.