Merge pull request TencentBlueKing#1108 from felixncheng/issue_1086

支持制品加固和下载请求转发

src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/arrowhead/ArrowheadScanExecutor.kt

@@ -75,8 +75,13 @@ class ArrowheadScanExecutor @Autowired constructor(
 
         // 执行扫描
         val result = dockerScanHelper.scan(
-            containerConfig.image, Binds(tmpBind, bind), listOf(containerConfig.args),
-            scannerInputFile, task
+            image = containerConfig.image,
+            binds=Binds(tmpBind, bind),
+            args = listOf(containerConfig.args),
+            scannerInputFile = scannerInputFile,
+            task = task,
+            userName = containerConfig.dockerRegistryUsername,
+            password = containerConfig.dockerRegistryPassword
         )
         if (!result) {
             return scanStatus(task, taskWorkDir, SubScanTaskStatus.TIMEOUT)

src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/scancodeCheck/ScancodeToolkitExecutor.kt

@@ -89,7 +89,9 @@ class ScancodeToolkitExecutor @Autowired constructor(
             binds = Binds(Bind(taskWorkDir.absolutePath, Volume(containerConfig.workDir))),
             args = containerCmd,
             scannerInputFile = scannerInputFile,
-            task = task
+            task = task,
+            userName = containerConfig.dockerRegistryUsername,
+            password = containerConfig.dockerRegistryPassword
         )
         if (!result) {
             return scanStatus(task, taskWorkDir, SubScanTaskStatus.TIMEOUT)

src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/standard/StandardScanExecutor.kt

@@ -77,7 +77,9 @@ class StandardScanExecutor(
             binds = Binds(Bind(taskWorkDir.absolutePath, Volume(CONTAINER_WORK_DIR))),
             args = args,
             scannerInputFile = scannerInputFile,
-            task = task
+            task = task,
+            userName = scanner.dockerRegistryUsername,
+            password = scanner.dockerRegistryPassword
         )
         return if (result) {
             SubScanTaskStatus.SUCCESS
@@ -121,7 +123,8 @@ class StandardScanExecutor(
             task.repoType,
             scannerInputFile.length(),
             task.packageKey,
-            task.packageVersion
+            task.packageVersion,
+            task.extra
         )
         val toolInput = ToolInput.create(
             task.taskId, convertToContainerPath(scannerInputFile.absolutePath, workDir), sha256, args

src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/trivy/TrivyScanExecutor.kt

@@ -103,7 +103,13 @@ class TrivyScanExecutor @Autowired constructor(
         val cacheBind = Bind(cacheDir.absolutePath, Volume(CACHE_DIR))
         val cmd = buildScanCmds(task, scannerInputFile)
         val result = dockerScanHelper.scan(
-            containerConfig.image, Binds(bind, cacheBind), cmd, scannerInputFile, task
+            image = containerConfig.image,
+            binds = Binds(bind, cacheBind),
+            args = cmd,
+            scannerInputFile = scannerInputFile,
+            task = task,
+            userName = containerConfig.dockerRegistryUsername,
+            password = containerConfig.dockerRegistryPassword
         )
         if (!result) {
             return scanStatus(task, taskWorkDir, SubScanTaskStatus.TIMEOUT)

src/backend/analysis-executor/biz-analysis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/util/DockerScanHelper.kt

@@ -51,6 +51,8 @@ class DockerScanHelper(
 
     fun scan(
         image: String,
+        userName: String?,
+        password: String?,
         binds: Binds,
         args: List<String>,
         scannerInputFile: File,
@@ -60,8 +62,13 @@ class DockerScanHelper(
         // 创建容器
         val maxFileSize = maxFileSize(scannerInputFile.length())
         val hostConfig = DockerUtils.dockerHostConfig(binds, maxFileSize, task.scanner.memory)
-        val containerId = dockerClient.createContainer(image, hostConfig, args)
-
+        val containerId = dockerClient.createContainer(
+            image = image,
+            hostConfig = hostConfig,
+            cmd = args,
+            userName = userName,
+            password = password
+        )
         taskContainerIdMap[task.taskId] = containerId
         logger.info(CommonUtils.buildLogMsg(task, "run container instance Id [$containerId]"))
         try {
@@ -70,7 +77,8 @@ class DockerScanHelper(
             val containerLogs = getContainerLogs(containerId)
             logger.info(
                 CommonUtils.buildLogMsg(
-                    task, "task docker run result[$result], [$containerId], logs:\n $containerLogs"
+                    task,
+                    "task docker run result[$result], [$containerId], logs:\n $containerLogs"
                 )
             )
             return result

src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/api/ScanClient.kt

@@ -98,4 +98,18 @@ interface ScanClient {
         @ApiParam(value = "许可证唯一标识集合")
         @RequestBody licenseIds: List<String>
     ): Response<Map<String, SpdxLicenseInfo>>
+
+    /**
+     * 校验token
+     *
+     * @return 通过返回true,否则返回false
+     * */
+    @GetMapping("/token/verify")
+    fun verifyToken(@RequestParam subtaskId: String, @RequestParam token: String): Response<Boolean>
+
+    /**
+     * 查询task状态
+     * */
+    @GetMapping("/task/{taskId}")
+    fun getTask(@PathVariable taskId: String): Response<ScanTask>
 }

src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/controller/ScanController.kt

@@ -33,19 +33,24 @@ import com.tencent.bkrepo.analyst.pojo.ScanTriggerType
 import com.tencent.bkrepo.analyst.pojo.SubScanTask
 import com.tencent.bkrepo.analyst.pojo.license.SpdxLicenseInfo
 import com.tencent.bkrepo.analyst.pojo.request.ReportResultRequest
-import com.tencent.bkrepo.common.api.pojo.Response
-import com.tencent.bkrepo.common.service.util.ResponseBuilder
 import com.tencent.bkrepo.analyst.pojo.request.ScanRequest
 import com.tencent.bkrepo.analyst.service.ScanService
+import com.tencent.bkrepo.analyst.service.ScanTaskService
 import com.tencent.bkrepo.analyst.service.SpdxLicenseService
+import com.tencent.bkrepo.analyst.service.TemporaryScanTokenService
+import com.tencent.bkrepo.common.api.pojo.Response
+import com.tencent.bkrepo.common.security.exception.AuthenticationException
 import com.tencent.bkrepo.common.security.util.SecurityUtils
+import com.tencent.bkrepo.common.service.util.ResponseBuilder
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.web.bind.annotation.RestController
 
 @RestController
 class ScanController @Autowired constructor(
     private val scanService: ScanService,
-    private val licenseService: SpdxLicenseService
+    private val licenseService: SpdxLicenseService,
+    private val tokenService: TemporaryScanTokenService,
+    private val scanTaskService: ScanTaskService,
 ) : ScanClient {
 
     override fun scan(scanRequest: ScanRequest): Response<ScanTask> {
@@ -75,4 +80,18 @@ class ScanController @Autowired constructor(
     override fun licenseInfoByIds(licenseIds: List<String>): Response<Map<String, SpdxLicenseInfo>> {
         return ResponseBuilder.success(licenseService.listLicenseByIds(licenseIds))
     }
+
+    override fun verifyToken(subtaskId: String, token: String): Response<Boolean> {
+        val ret = try {
+            tokenService.checkToken(subtaskId, token)
+            true
+        } catch (e: AuthenticationException) {
+            false
+        }
+        return ResponseBuilder.success(ret)
+    }
+
+    override fun getTask(taskId: String): Response<ScanTask> {
+        return ResponseBuilder.success(scanTaskService.task(taskId))
+    }
 }

src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/controller/user/UserTemporaryScanController.kt

@@ -27,12 +27,12 @@
 
 package com.tencent.bkrepo.analyst.controller.user
 
-import com.tencent.bkrepo.common.api.pojo.Response
-import com.tencent.bkrepo.common.analysis.pojo.scanner.standard.ToolInput
-import com.tencent.bkrepo.common.service.util.ResponseBuilder
 import com.tencent.bkrepo.analyst.pojo.request.ReportResultRequest
 import com.tencent.bkrepo.analyst.service.ScanService
 import com.tencent.bkrepo.analyst.service.TemporaryScanTokenService
+import com.tencent.bkrepo.common.analysis.pojo.scanner.standard.ToolInput
+import com.tencent.bkrepo.common.api.pojo.Response
+import com.tencent.bkrepo.common.service.util.ResponseBuilder
 import io.swagger.annotations.Api
 import io.swagger.annotations.ApiOperation
 import org.springframework.web.bind.annotation.GetMapping
@@ -59,7 +59,7 @@ class UserTemporaryScanController(
         @RequestParam token: String
     ): Response<ToolInput> {
         temporaryScanTokenService.checkToken(subtaskId, token)
-        return ResponseBuilder.success(temporaryScanTokenService.getToolInput(subtaskId))
+        return ResponseBuilder.success(temporaryScanTokenService.getToolInput(subtaskId, token))
     }
 
     @ApiOperation("拉取扫描子任务")
@@ -69,7 +69,7 @@ class UserTemporaryScanController(
         @RequestParam token: String
     ): Response<ToolInput?> {
         temporaryScanTokenService.checkToken(executionCluster, token)
-        val toolInput = temporaryScanTokenService.pullToolInput(executionCluster)
+        val toolInput = temporaryScanTokenService.pullToolInput(executionCluster, token)
         toolInput?.let { temporaryScanTokenService.setToken(it.taskId, token) }
         return ResponseBuilder.success(toolInput)
     }
@@ -84,7 +84,6 @@ class UserTemporaryScanController(
         return ResponseBuilder.success()
     }
 
-
     @ApiOperation("扫描任务状态更新")
     @PutMapping("/scan/subtask/{subtaskId}/status")
     fun updateSubScanTaskStatus(

src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dispatcher/DockerDispatcher.kt

@@ -58,7 +58,11 @@ class DockerDispatcher(
     private val subScanTaskDao: SubScanTaskDao,
     private val redisTemplate: ObjectProvider<RedisTemplate<String, String>>
 ) : SubtaskPushDispatcher<DockerExecutionCluster>(
-    executionCluster, scannerProperties, scanService, subtaskStateMachine, temporaryScanTokenService
+    executionCluster,
+    scannerProperties,
+    scanService,
+    subtaskStateMachine,
+    temporaryScanTokenService
 ) {
 
     private val dockerClient by lazy {
@@ -94,7 +98,11 @@ class DockerDispatcher(
                 heartbeatTimeout = scannerProperties.heartbeatTimeout
             )
             val containerId = dockerClient.createContainer(
-                image = scanner.image, hostConfig = hostConfig(), cmd = command
+                image = scanner.image,
+                userName = scanner.dockerRegistryUsername,
+                password = scanner.dockerRegistryPassword,
+                hostConfig = hostConfig(),
+                cmd = command
             )
             dockerClient.startContainerCmd(containerId).exec()
             redisTemplate.ifAvailable

src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dispatcher/KubernetesDeploymentDispatcher.kt

@@ -3,6 +3,7 @@ package com.tencent.bkrepo.analyst.dispatcher
 import com.tencent.bkrepo.analyst.configuration.ScannerProperties
 import com.tencent.bkrepo.analyst.dao.SubScanTaskDao
 import com.tencent.bkrepo.analyst.dispatcher.dsl.addContainerItem
+import com.tencent.bkrepo.analyst.dispatcher.dsl.addImagePullSecretsItemIfNeed
 import com.tencent.bkrepo.analyst.dispatcher.dsl.limits
 import com.tencent.bkrepo.analyst.dispatcher.dsl.metadata
 import com.tencent.bkrepo.analyst.dispatcher.dsl.requests
@@ -169,6 +170,7 @@ class KubernetesDeploymentDispatcher(
                             name = deploymentName
                             image = scanner.image
                             command = cmd
+                            addImagePullSecretsItemIfNeed(scanner, k8sProps)
                             resources {
                                 limits(
                                     cpu = k8sProps.limitCpu,

src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dispatcher/KubernetesDispatcher.kt

@@ -29,6 +29,7 @@ package com.tencent.bkrepo.analyst.dispatcher
 
 import com.tencent.bkrepo.analyst.configuration.ScannerProperties
 import com.tencent.bkrepo.analyst.dispatcher.dsl.addContainerItem
+import com.tencent.bkrepo.analyst.dispatcher.dsl.addImagePullSecretsItemIfNeed
 import com.tencent.bkrepo.analyst.dispatcher.dsl.limits
 import com.tencent.bkrepo.analyst.dispatcher.dsl.metadata
 import com.tencent.bkrepo.analyst.dispatcher.dsl.requests
@@ -59,7 +60,11 @@ class KubernetesDispatcher(
     subtaskStateMachine: StateMachine,
     temporaryScanTokenService: TemporaryScanTokenService,
 ) : SubtaskPushDispatcher<KubernetesJobExecutionCluster>(
-    executionCluster, scannerProperties, scanService, subtaskStateMachine, temporaryScanTokenService
+    executionCluster,
+    scannerProperties,
+    scanService,
+    subtaskStateMachine,
+    temporaryScanTokenService
 ) {
 
     private val client by lazy { createClient(executionCluster.kubernetesProperties) }
@@ -163,6 +168,7 @@ class KubernetesDispatcher(
                             name = jobName
                             image = containerImage
                             command = cmd
+                            addImagePullSecretsItemIfNeed(scanner, k8sProps)
                             resources {
                                 requests(
                                     cpu = k8sProps.requestCpu,
@@ -219,7 +225,14 @@ class KubernetesDispatcher(
         return ignoreApiException {
             val namespace = executionCluster.kubernetesProperties.namespace
             batchV1Api.deleteNamespacedJob(
-                jobName, namespace, null, null, null, null, "Foreground", null
+                jobName,
+                namespace,
+                null,
+                null,
+                null,
+                null,
+                "Foreground",
+                null
             )
             logger.info("job[$jobName] clean success")
             true

src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dispatcher/dsl/PodDsl.kt

@@ -1,7 +1,11 @@
 package com.tencent.bkrepo.analyst.dispatcher.dsl
 
+import com.tencent.bkrepo.analyst.pojo.execution.KubernetesExecutionClusterProperties
+import com.tencent.bkrepo.analyst.utils.ScannerUtil
+import com.tencent.bkrepo.common.analysis.pojo.scanner.standard.StandardScanner
 import io.kubernetes.client.custom.Quantity
 import io.kubernetes.client.openapi.models.V1Container
+import io.kubernetes.client.openapi.models.V1LocalObjectReference
 import io.kubernetes.client.openapi.models.V1ObjectMeta
 import io.kubernetes.client.openapi.models.V1PodSpec
 import io.kubernetes.client.openapi.models.V1PodTemplateSpec
@@ -69,3 +73,14 @@ fun V1ResourceRequirements.limits(cpu: Double, memory: Long, ephemeralStorage: L
         )
     )
 }
+
+fun V1PodSpec.addImagePullSecretsItemIfNeed(
+    scanner: StandardScanner,
+    k8sClusterProp: KubernetesExecutionClusterProperties
+) {
+    if (ScannerUtil.isPrivateImage(scanner)) {
+        val secret = ScannerUtil.getOrCreateSecret(scanner, k8sClusterProp)
+        val secretName = secret.metadata!!.name
+        addImagePullSecretsItem(V1LocalObjectReference().name(secretName))
+    }
+}

src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dispatcher/dsl/SecretDsl.kt

@@ -0,0 +1,48 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.analyst.dispatcher.dsl
+
+import io.kubernetes.client.openapi.models.V1ObjectMeta
+import io.kubernetes.client.openapi.models.V1Secret
+
+/**
+ * 创建Secret并配置
+ */
+fun V1Secret(configuration: V1Secret.() -> Unit): V1Secret {
+    return V1Secret().apply(configuration)
+}
+
+/**
+ * 配置Secret元数据
+ */
+fun V1Secret.metadata(configuration: V1ObjectMeta.() -> Unit) {
+    if (metadata == null) {
+        metadata = V1ObjectMeta()
+    }
+    metadata!!.configuration()
+}

src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/TemporaryScanTokenService.kt

@@ -36,6 +36,6 @@ interface TemporaryScanTokenService {
     fun createExecutionClusterToken(executionClusterName: String): String
     fun checkToken(subtaskId: String, token: String?)
     fun deleteToken(subtaskId: String)
-    fun getToolInput(subtaskId: String): ToolInput
-    fun pullToolInput(executionCluster: String): ToolInput?
+    fun getToolInput(subtaskId: String, token: String): ToolInput
+    fun pullToolInput(executionCluster: String, token: String): ToolInput?
 }

src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanTaskServiceImpl.kt

@@ -107,8 +107,11 @@ class ScanTaskServiceImpl(
 
     override fun task(taskId: String): ScanTask {
         return scanTaskDao.findById(taskId)?.let { task ->
+            val repos = RuleUtil.getRepoNames(task.rule?.readJsonString())
             if (task.projectId == null) {
                 permissionCheckHandler.permissionManager.checkPrincipal(SecurityUtils.getUserId(), PrincipalType.ADMIN)
+            } else if (repos.isNotEmpty()) {
+                permissionCheckHandler.checkReposPermission(task.projectId, repos, PermissionAction.READ)
             } else {
                 permissionCheckHandler.checkProjectPermission(task.projectId, PermissionAction.MANAGE)
             }