Add kube strategy

Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
seqeralabs · Sep 19, 2024 · b2f3e96 · b2f3e96
1 parent c82f403
commit b2f3e96
Show file tree

Hide file tree

Showing 13 changed files with 233 additions and 15 deletions.
diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy
@@ -77,7 +77,7 @@ class KubeBuildStrategy extends BuildStrategy {
             Files.write(configFile, JsonOutput.prettyPrint(req.configJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING)
         }
         // save remote files for singularity
-        if( req.configJson && req.formatSingularity()) {
+        if( req.configJson && req.formatSingularity() ) {
             final remoteFile = req.workDir.resolve('singularity-remote.yaml')
             final content = RegHelper.singularityRemoteFile(req.targetImage)
             Files.write(remoteFile, content.bytes, CREATE, WRITE, TRUNCATE_EXISTING)

diff --git a/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy
@@ -28,7 +28,7 @@ import io.seqera.wave.service.builder.BuildRequest
 import io.seqera.wave.service.builder.BuildStrategy
 import io.seqera.wave.service.cleanup.CleanupService
 import io.seqera.wave.service.mirror.MirrorRequest
-import io.seqera.wave.service.mirror.MirrorStrategy
+import io.seqera.wave.service.mirror.strategy.MirrorStrategy
 import io.seqera.wave.service.scan.ScanRequest
 import io.seqera.wave.service.scan.ScanStrategy
 import jakarta.inject.Inject

diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy
@@ -26,6 +26,8 @@ import io.kubernetes.client.openapi.models.V1Pod
 import io.kubernetes.client.openapi.models.V1PodList
 import io.seqera.wave.configuration.BlobCacheConfig
 import io.seqera.wave.configuration.ScanConfig
+import io.seqera.wave.service.mirror.MirrorConfig
+
 /**
  * Defines Kubernetes operations
  *
@@ -68,6 +70,8 @@ interface K8sService {
 
     V1Job launchScanJob(String name, String containerImage, List<String> args, Path workDir, Path creds, ScanConfig scanConfig, Map<String,String> nodeSelector)
 
+    V1Job launchMirrorJob(String name, String containerImage, List<String> args, Path workDir, Path creds, MirrorConfig config)
+
     @Deprecated
     V1PodList waitJob(V1Job job, Long timeout)
 

diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy
@@ -48,6 +48,7 @@ import io.seqera.wave.configuration.BlobCacheConfig
 import io.seqera.wave.configuration.BuildConfig
 import io.seqera.wave.configuration.ScanConfig
 import io.seqera.wave.core.ContainerPlatform
+import io.seqera.wave.service.mirror.MirrorConfig
 import io.seqera.wave.service.scan.Trivy
 import jakarta.inject.Inject
 import jakarta.inject.Singleton
@@ -782,6 +783,67 @@ class K8sServiceImpl implements K8sService {
         return builder.build()
     }
 
+    @Override
+    V1Job launchMirrorJob(String name, String containerImage, List<String> args, Path workDir, Path creds, MirrorConfig config) {
+        final spec = mirrorJobSpec(name, containerImage, args, workDir, creds, config)
+        return k8sClient
+                .batchV1Api()
+                .createNamespacedJob(namespace, spec, null, null, null,null)
+    }
+
+    V1Job mirrorJobSpec(String name, String containerImage, List<String> args, Path workDir, Path credsFile, MirrorConfig config) {
+
+        // required volumes
+        final mounts = new ArrayList<V1VolumeMount>(5)
+        mounts.add(mountBuildStorage(workDir, storageMountPath, true))
+
+        final volumes = new ArrayList<V1Volume>(5)
+        volumes.add(volumeBuildStorage(storageMountPath, storageClaimName))
+
+        if( credsFile ){
+            mounts.add(0, mountHostPath(credsFile, storageMountPath, '/tmp/config.json'))
+        }
+
+        V1JobBuilder builder = new V1JobBuilder()
+
+        //metadata section
+        builder.withNewMetadata()
+                .withNamespace(namespace)
+                .withName(name)
+                .addToLabels(labels)
+                .endMetadata()
+
+        //spec section
+        def spec = builder
+                .withNewSpec()
+                .withBackoffLimit(config.retryAttempts)
+                .withNewTemplate()
+                .editOrNewSpec()
+                .withServiceAccount(serviceAccount)
+                .withRestartPolicy("Never")
+                .addAllToVolumes(volumes)
+
+        final requests = new V1ResourceRequirements()
+        if( config.requestsCpu )
+            requests.putRequestsItem('cpu', new Quantity(config.requestsCpu))
+        if( config.requestsMemory )
+            requests.putRequestsItem('memory', new Quantity(config.requestsMemory))
+
+        // container section
+        final container = new V1ContainerBuilder()
+                .withName(name)
+                .withImage(containerImage)
+                .withArgs(args)
+                .withVolumeMounts(mounts)
+                .withResources(requests)
+                .withEnv(new V1EnvVar().name("REGISTRY_AUTH_FILE").value("/tmp/config.json"))
+
+        // spec section
+        spec.withContainers(container.build()).endSpec().endTemplate().endSpec()
+
+        return builder.build()
+    }
+
     protected List<V1EnvVar> toEnvList(Map<String,String> env) {
         final result = new ArrayList<V1EnvVar>(env.size())
         for( Map.Entry<String,String> it : env )

diff --git a/...or/impl/ContainerMirrorServiceImpl.groovy → .../mirror/ContainerMirrorServiceImpl.groovy b/...or/impl/ContainerMirrorServiceImpl.groovy → .../mirror/ContainerMirrorServiceImpl.groovy
@@ -16,7 +16,7 @@
  *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package io.seqera.wave.service.mirror.impl
+package io.seqera.wave.service.mirror
 
 import java.util.concurrent.CompletableFuture
 import java.util.concurrent.ExecutorService
@@ -29,10 +29,6 @@ import io.seqera.wave.service.job.JobHandler
 import io.seqera.wave.service.job.JobService
 import io.seqera.wave.service.job.JobSpec
 import io.seqera.wave.service.job.JobState
-import io.seqera.wave.service.mirror.ContainerMirrorService
-import io.seqera.wave.service.mirror.MirrorRequest
-import io.seqera.wave.service.mirror.MirrorState
-import io.seqera.wave.service.mirror.MirrorStateStore
 import io.seqera.wave.service.persistence.PersistenceService
 import jakarta.inject.Inject
 import jakarta.inject.Named

diff --git a/src/main/groovy/io/seqera/wave/service/mirror/MirrorConfig.groovy b/src/main/groovy/io/seqera/wave/service/mirror/MirrorConfig.groovy
@@ -22,6 +22,7 @@ import java.time.Duration
 
 import groovy.transform.CompileStatic
 import io.micronaut.context.annotation.Value
+import io.micronaut.core.annotation.Nullable
 import jakarta.inject.Singleton
 
 /**
@@ -44,4 +45,16 @@ class MirrorConfig {
 
     @Value('${wave.mirror.skopeoImage:`quay.io/skopeo/stable`}')
     String skopeoImage
+
+    @Value('${wave.mirror.retry-attempts:3}')
+    Integer retryAttempts
+
+    @Nullable
+    @Value('${wave.mirror.requestsCpu}')
+    String requestsCpu
+
+    @Nullable
+    @Value('${wave.mirror.requestsMemory}')
+    String requestsMemory
+
 }
diff --git a/...e/mirror/impl/DockerMirrorStrategy.groovy → ...rror/strategy/DockerMirrorStrategy.groovy b/...e/mirror/impl/DockerMirrorStrategy.groovy → ...rror/strategy/DockerMirrorStrategy.groovy
@@ -16,7 +16,7 @@
  *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package io.seqera.wave.service.mirror.impl
+package io.seqera.wave.service.mirror.strategy
 
 
 import java.nio.file.Files
@@ -28,7 +28,6 @@ import groovy.util.logging.Slf4j
 import io.micronaut.context.annotation.Value
 import io.seqera.wave.service.mirror.MirrorConfig
 import io.seqera.wave.service.mirror.MirrorRequest
-import io.seqera.wave.service.mirror.MirrorStrategy
 import jakarta.inject.Inject
 import jakarta.inject.Singleton
 import static java.nio.file.StandardOpenOption.CREATE
@@ -56,7 +55,6 @@ class DockerMirrorStrategy extends MirrorStrategy {
         Path configFile = null
         // create the work directory
         Files.createDirectories(request.workDir)
-
         // save docker config for creds
         if( request.authJson ) {
             configFile = request.workDir.resolve('config.json')

diff --git a/src/main/groovy/io/seqera/wave/service/mirror/strategy/KubeMirrorStrategy.groovy b/src/main/groovy/io/seqera/wave/service/mirror/strategy/KubeMirrorStrategy.groovy
@@ -0,0 +1,83 @@
+/*
+ *  Wave, containers provisioning service
+ *  Copyright (c) 2023-2024, Seqera Labs
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package io.seqera.wave.service.mirror.strategy
+
+import java.nio.file.Files
+import java.nio.file.Path
+
+import groovy.json.JsonOutput
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import io.kubernetes.client.openapi.ApiException
+import io.micronaut.context.annotation.Primary
+import io.micronaut.context.annotation.Requires
+import io.seqera.wave.exception.BadRequestException
+import io.seqera.wave.service.k8s.K8sService
+import io.seqera.wave.service.mirror.MirrorConfig
+import io.seqera.wave.service.mirror.MirrorRequest
+import jakarta.inject.Inject
+import jakarta.inject.Singleton
+import static java.nio.file.StandardOpenOption.CREATE
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING
+import static java.nio.file.StandardOpenOption.WRITE
+
+/**
+ * Implements a container mirror runner based on Kubernetes
+ *
+ * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
+ */
+@Slf4j
+@Primary
+@Requires(property = 'wave.build.k8s')
+@Singleton
+@CompileStatic
+class KubeMirrorStrategy extends MirrorStrategy {
+
+    @Inject
+    private MirrorConfig config
+
+    @Inject
+    private K8sService k8sService
+
+    @Override
+    void mirrorJob(String jobName, MirrorRequest request) {
+        Path configFile = null
+        // create the work directory
+        Files.createDirectories(request.workDir)
+        // save docker config for creds
+        if( request.authJson ) {
+            configFile = request.workDir.resolve('config.json')
+            Files.write(configFile, JsonOutput.prettyPrint(request.authJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING)
+        }
+
+        try {
+            k8sService.launchMirrorJob(
+                    jobName,
+                    config.skopeoImage,
+                    copyCommand(request),
+                    request.workDir,
+                    configFile,
+                    config)
+        }
+        catch (ApiException e) {
+            throw new BadRequestException("Unexpected build failure - ${e.responseBody}", e)
+        }
+
+    }
+}
diff --git a/...wave/service/mirror/MirrorStrategy.groovy → ...ice/mirror/strategy/MirrorStrategy.groovy b/...wave/service/mirror/MirrorStrategy.groovy → ...ice/mirror/strategy/MirrorStrategy.groovy
@@ -16,10 +16,11 @@
  *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package io.seqera.wave.service.mirror
+package io.seqera.wave.service.mirror.strategy
 
-import groovy.transform.CompileStatic
 
+import groovy.transform.CompileStatic
+import io.seqera.wave.service.mirror.MirrorRequest
 /**
  * Implement the common strategy to handle container mirror
  * via Skopeo

diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy
@@ -42,6 +42,7 @@ import io.micronaut.context.annotation.Replaces
 import io.micronaut.test.extensions.spock.annotation.MicronautTest
 import io.seqera.wave.configuration.BlobCacheConfig
 import io.seqera.wave.configuration.ScanConfig
+import io.seqera.wave.service.mirror.MirrorConfig
 /**
  *
  * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
@@ -828,6 +829,65 @@ class K8sServiceImplTest extends Specification {
         ctx.close()
     }
 
+    def 'should create mirror job spec'() {
+        given:
+        def PROPS = [
+                'wave.build.workspace': '/build/work',
+                'wave.build.k8s.namespace': 'foo',
+                'wave.build.k8s.configPath': '/home/kube.config',
+                'wave.build.k8s.storage.claimName': 'bar',
+                'wave.build.k8s.storage.mountPath': '/build',
+                'wave.build.k8s.service-account': 'theAdminAccount',
+                'wave.mirror.retry-attempts': 3
+        ]
+        and:
+        def ctx = ApplicationContext.run(PROPS)
+        def k8sService = ctx.getBean(K8sServiceImpl)
+        def name = 'scan-job'
+        def containerImage = 'scan-image:latest'
+        def args = ['arg1', 'arg2']
+        def workDir = Path.of('/build/work/dir')
+        def credsFile = Path.of('/build/work/dir/creds/file')
+        def mirrorConfig = Mock(MirrorConfig) {
+            getRequestsCpu() >> null
+            getRequestsMemory() >> null
+            getRetryAttempts() >> 3
+        }
+
+        when:
+        def job = k8sService.mirrorJobSpec(name, containerImage, args, workDir, credsFile, mirrorConfig)
+
+        then:
+        job.metadata.name == name
+        job.metadata.namespace == 'foo'
+        job.spec.backoffLimit == 3
+        job.spec.template.spec.containers[0].image == containerImage
+        job.spec.template.spec.containers[0].args == args
+        job.spec.template.spec.containers[0].resources.requests == null
+        job.spec.template.spec.containers[0].env == [new V1EnvVar().name('REGISTRY_AUTH_FILE').value('/tmp/config.json')]
+        and:
+        job.spec.template.spec.containers[0].volumeMounts.size() == 2
+        and:
+        with(job.spec.template.spec.containers[0].volumeMounts[0]) {
+            mountPath == '/tmp/config.json'
+            readOnly == true
+            subPath == 'work/dir/creds/file'
+        }
+        and:
+        with(job.spec.template.spec.containers[0].volumeMounts[1]) {
+            mountPath == '/build/work/dir'
+            readOnly == true
+            subPath == 'work/dir'
+        }
+        and:
+        job.spec.template.spec.volumes.size() == 1
+        job.spec.template.spec.volumes[0].persistentVolumeClaim.claimName == 'bar'
+        job.spec.template.spec.restartPolicy == 'Never'
+
+        cleanup:
+        ctx.close()
+    }
+
     def 'should create scan job spec without resource requests'() {
         given:
         def PROPS = [

diff --git a/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy
@@ -33,7 +33,6 @@ import io.seqera.wave.core.ContainerPlatform
 import io.seqera.wave.service.inspect.ContainerInspectService
 import io.seqera.wave.service.job.JobSpec
 import io.seqera.wave.service.job.JobState
-import io.seqera.wave.service.mirror.impl.ContainerMirrorServiceImpl
 import io.seqera.wave.service.persistence.PersistenceService
 import io.seqera.wave.tower.PlatformId
 import jakarta.inject.Inject

diff --git a/...rror/impl/DockerMirrorStrategyTest.groovy → .../strategy/DockerMirrorStrategyTest.groovy b/...rror/impl/DockerMirrorStrategyTest.groovy → .../strategy/DockerMirrorStrategyTest.groovy
@@ -16,7 +16,7 @@
  *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package io.seqera.wave.service.mirror.impl
+package io.seqera.wave.service.mirror.strategy
 
 import spock.lang.Specification
 

diff --git a/.../service/mirror/MirrorStrategyTest.groovy → ...mirror/strategy/MirrorStrategyTest.groovy b/.../service/mirror/MirrorStrategyTest.groovy → ...mirror/strategy/MirrorStrategyTest.groovy
@@ -16,7 +16,9 @@
  *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package io.seqera.wave.service.mirror
+package io.seqera.wave.service.mirror.strategy
+
+import io.seqera.wave.service.mirror.MirrorRequest
 
 import spock.lang.Specification
 import spock.lang.Unroll