feat: 增加自动恢复功能 TencentBlueKing#2337

* feat: 增加自动恢复功能 TencentBlueKing#2337

* feat: 统计任务锁时间变更 TencentBlueKing#2337

* feat:  代码调整TencentBlueKing#2337

* feat:  引用清理调整TencentBlueKing#2337

* feat:  删除多余代码TencentBlueKing#2337

* feat:  package不进行最后修改时间判断TencentBlueKing#2337

src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/base/EventType.kt

@@ -51,6 +51,7 @@ enum class EventType(val msgKey: String) {
     NODE_DOWNLOADED("artifact.event.node-downloaded"),
     NODE_CLEAN("artifact.event.node-clean"),
     NODE_UPDATE_ACCESS_DATE("artifact.event.node-update-access-date"),
+    NODE_SEPARATION_RECOVERY("artifact.event.node-separation-recovery"),
 
     // METADATA
     METADATA_DELETED("artifact.event.metadata-deleted"),

src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/node/NodeSeparationRecoveryEvent.kt

@@ -0,0 +1,49 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.artifact.event.node
+
+import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent
+import com.tencent.bkrepo.common.artifact.event.base.EventType
+
+/**
+ * 将冷节点自动恢复
+ */
+class NodeSeparationRecoveryEvent(
+    override val projectId: String,
+    override val repoName: String,
+    override val resourceKey: String,
+    override val userId: String,
+    val repoType: String,
+) : ArtifactEvent(
+    type = EventType.NODE_SEPARATION_RECOVERY,
+    projectId = projectId,
+    repoName = repoName,
+    resourceKey = resourceKey,
+    userId = userId,
+    data = mapOf("repoType" to repoType)
+)

src/backend/job/biz-job/build.gradle.kts

@@ -39,6 +39,7 @@ dependencies {
     implementation(project(":repository:api-repository"))
     implementation(project(":helm:api-helm"))
     implementation(project(":oci:api-oci"))
+    implementation(project(":maven:api-maven"))
     implementation(project(":replication:api-replication"))
     implementation(project(":archive:api-archive"))
     implementation(project(":common:common-operate:operate-service"))

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt

@@ -106,7 +106,6 @@ const val SEPARATE = "SEPARATE"
 const val PACKAGE_COLLECTION_NAME = "package"
 const val PACKAGE_VERSION_COLLECTION_NAME = "package_version"
 const val PACKAGE_DOWNLOADS_COLLECTION_NAME = "package_downloads"
-const val FILE_REFERENCE_COLLECTION_NAME = "file_reference"
 const val SEPARATION_TASK_COLLECTION_NAME = "separation_task"
 
 const val PACKAGE_VERSION = "version"

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt

@@ -95,7 +95,7 @@ class ActiveProjectEmptyFolderCleanupJob(
     }
 
     override fun getLockAtMostFor(): Duration {
-        return Duration.ofDays(1)
+        return Duration.ofDays(14)
     }
 
     override fun createJobContext(): EmptyFolderCleanupJobContext {

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt

@@ -132,7 +132,7 @@ class ActiveProjectNodeFolderStatJob(
     /**
      * 最长加锁时间
      */
-    override fun getLockAtMostFor(): Duration = Duration.ofDays(1)
+    override fun getLockAtMostFor(): Duration = Duration.ofDays(14)
 
     companion object {
         private val logger = LoggerFactory.getLogger(ActiveProjectNodeFolderStatJob::class.java)

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt

@@ -108,7 +108,7 @@ class InactiveProjectEmptyFolderCleanupJob(
     }
 
     override fun getLockAtMostFor(): Duration {
-        return Duration.ofDays(1)
+        return Duration.ofDays(14)
     }
 
     override fun createJobContext(): EmptyFolderCleanupJobContext {

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectNodeFolderStatJob.kt

@@ -83,7 +83,7 @@ class InactiveProjectNodeFolderStatJob(
     /**
      * 最长加锁时间
      */
-    override fun getLockAtMostFor(): Duration = Duration.ofDays(1)
+    override fun getLockAtMostFor(): Duration = Duration.ofDays(14)
 
     fun statProjectCheck(
         projectId: String,

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ProjectRepoMetricsStatJob.kt

@@ -130,7 +130,7 @@ open class ProjectRepoMetricsStatJob(
     }
 
     override fun getLockAtMostFor(): Duration {
-        return Duration.ofDays(1)
+        return Duration.ofDays(14)
     }
 
     override fun onRunCollectionFinished(collectionName: String, context: JobContext) {

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/DataSeparationJobProperties.kt

@@ -36,5 +36,5 @@ class DataSeparationJobProperties(
     override var enabled: Boolean = false,
     override var cron: String = Scheduled.CRON_DISABLED,
     // 任务处于running 状态超过多久没有更新数据，则判断任务已经中断
-    var waitTime: Duration = Duration.ofMillis(60),
+    var waitTime: Duration = Duration.ofMinutes(120)
 ) : MongodbJobProperties(enabled)

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/config/DataSeparationConfig.kt

@@ -69,6 +69,8 @@ data class DataSeparationConfig(
     var restoreTaskConcurrency: Int = Runtime.getRuntime().availableProcessors() * 2,
     // 允许同时执行的fix任务数
     var fixTaskConcurrency: Int = Runtime.getRuntime().availableProcessors() * 2,
+    // 是否允许自动恢复
+    var enableAutoRecovery: Boolean = false
 )
 
 

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/dao/SeparationNodeDao.kt

@@ -64,6 +64,15 @@ import java.time.LocalDateTime
 @Repository
 class SeparationNodeDao : MonthRangeShardingMongoDao<TSeparationNode>() {
 
+    fun findOneByFullPath(
+        projectId: String, repoName: String,
+        fullPath: String, separationDate: LocalDateTime
+    ): TSeparationNode? {
+        return this.findOne(SeparationQueryHelper.fullPathQuery(
+            projectId, repoName, fullPath, separationDate
+        ))
+    }
+
     fun findOne(
         projectId: String, repoName: String,
         versionPath: String, separationDate: LocalDateTime

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/listener/SeparationRecoveryEventConsumer.kt

@@ -0,0 +1,196 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.separation.listener
+
+import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent
+import com.tencent.bkrepo.common.artifact.event.base.EventType
+import com.tencent.bkrepo.common.artifact.pojo.RepositoryType
+import com.tencent.bkrepo.job.RESTORE
+import com.tencent.bkrepo.job.separation.config.DataSeparationConfig
+import com.tencent.bkrepo.job.separation.dao.SeparationNodeDao
+import com.tencent.bkrepo.job.separation.dao.SeparationPackageDao
+import com.tencent.bkrepo.job.separation.dao.SeparationPackageVersionDao
+import com.tencent.bkrepo.job.separation.model.TSeparationNode
+import com.tencent.bkrepo.job.separation.model.TSeparationPackageVersion
+import com.tencent.bkrepo.job.separation.pojo.NodeFilterInfo
+import com.tencent.bkrepo.job.separation.pojo.PackageFilterInfo
+import com.tencent.bkrepo.job.separation.pojo.RecoveryNodeInfo
+import com.tencent.bkrepo.job.separation.pojo.RecoveryVersionInfo
+import com.tencent.bkrepo.job.separation.pojo.SeparationContent
+import com.tencent.bkrepo.job.separation.pojo.task.SeparationTaskRequest
+import com.tencent.bkrepo.job.separation.service.SeparationTaskService
+import com.tencent.bkrepo.job.separation.service.impl.repo.RepoSpecialSeparationMappings
+import org.slf4j.LoggerFactory
+import org.springframework.messaging.Message
+import org.springframework.stereotype.Component
+import java.time.format.DateTimeFormatter
+import java.util.function.Consumer
+
+/**
+ * 消费降冷自动恢复事件
+ */
+@Component("separationRecovery")
+class SeparationRecoveryEventConsumer(
+    private val separationTaskService: SeparationTaskService,
+    private val dataSeparationConfig: DataSeparationConfig,
+    private val separationPackageDao: SeparationPackageDao,
+    private val separationPackageVersionDao: SeparationPackageVersionDao,
+    private val separationNodeDao: SeparationNodeDao
+) : Consumer<Message<ArtifactEvent>> {
+
+    /**
+     * 允许接收的事件类型
+     */
+    private val acceptTypes = setOf(
+        EventType.NODE_SEPARATION_RECOVERY,
+    )
+
+    override fun accept(message: Message<ArtifactEvent>) {
+        if (!dataSeparationConfig.enableAutoRecovery) return
+        if (!acceptTypes.contains(message.payload.type)) {
+            return
+        }
+        logger.info("current separation recovery message header is ${message.headers}")
+        doSeparationRecovery(message.payload)
+    }
+
+    private fun doSeparationRecovery(event: ArtifactEvent) {
+        val recoveryNodeInfo = buildRecoveryNodeInfo(event)
+        val task = when (recoveryNodeInfo.repoType) {
+            RepositoryType.MAVEN.name -> {
+                val recoveryVersionInfo = RepoSpecialSeparationMappings.getRecoveryPackageVersionData(recoveryNodeInfo)
+                val version = recoveryVersionCheck(recoveryVersionInfo) ?: return
+                buildVersionSeparationTaskRequest(recoveryVersionInfo, version)
+            }
+            RepositoryType.GENERIC.name -> {
+                val node = recoveryNodeCheck(recoveryNodeInfo) ?: return
+                buildNodeSeparationTaskRequest(recoveryNodeInfo, node)
+            }
+            else -> {
+                null
+            }
+        }
+        if (task != null) {
+            separationTaskService.createSeparationTask(task)
+        }
+    }
+
+    private fun buildRecoveryNodeInfo(event: ArtifactEvent): RecoveryNodeInfo {
+        return RecoveryNodeInfo(
+            projectId = event.projectId,
+            repoName = event.repoName,
+            fullPath = event.resourceKey,
+            repoType = event.data["repoType"].toString()
+        )
+    }
+
+    private fun recoveryNodeCheck(recoveryNodeInfo: RecoveryNodeInfo): TSeparationNode? {
+        with(recoveryNodeInfo) {
+            val separateDates = separationTaskService.findDistinctSeparationDate(projectId, repoName)
+            if (separateDates.isEmpty()) return null
+            separateDates.forEach {
+                val separationNode = separationNodeDao.findOneByFullPath(projectId, repoName, fullPath, it)
+                if (separationNode != null) {
+                    return separationNode
+                }
+            }
+            return null
+        }
+    }
+
+    private fun recoveryVersionCheck(recoveryVersionInfo: RecoveryVersionInfo): TSeparationPackageVersion? {
+        with(recoveryVersionInfo) {
+            val separateDates = separationTaskService.findDistinctSeparationDate(projectId, repoName)
+            if (separateDates.isEmpty()) return null
+            val separationPackage = separationPackageDao.findByKey(projectId, repoName, packageKey) ?: return null
+            separateDates.forEach {
+                val version = separationPackageVersionDao.findByName(separationPackage.id!!, version, it)
+                if (version != null) {
+                    return version
+                }
+            }
+            return null
+        }
+    }
+
+    private fun buildVersionSeparationTaskRequest(
+        recoveryVersionInfo: RecoveryVersionInfo,
+        separationPackageVersion: TSeparationPackageVersion
+    ): SeparationTaskRequest {
+        return SeparationTaskRequest(
+            projectId = recoveryVersionInfo.projectId,
+            repoName = recoveryVersionInfo.repoName,
+            type = RESTORE,
+            separateAt = separationPackageVersion.separationDate.format(DateTimeFormatter.ISO_DATE_TIME),
+            content = buildSeparationContent(recoveryVersionInfo.packageKey, separationPackageVersion.name)!!
+        )
+    }
+
+    private fun buildNodeSeparationTaskRequest(
+        recoveryNodeInfo: RecoveryNodeInfo,
+        separationNode: TSeparationNode
+    ): SeparationTaskRequest {
+        return SeparationTaskRequest(
+            projectId = recoveryNodeInfo.projectId,
+            repoName = recoveryNodeInfo.repoName,
+            type = RESTORE,
+            separateAt = separationNode.separationDate.format(DateTimeFormatter.ISO_DATE_TIME),
+            content = buildSeparationContent(fullPath = separationNode.fullPath)!!
+        )
+    }
+
+    private fun buildSeparationContent(
+        packageKey: String? = null,
+        version: String? = null,
+        fullPath: String? = null
+    ): SeparationContent? {
+        if (packageKey.isNullOrEmpty() && version.isNullOrEmpty() && fullPath.isNullOrEmpty()) return null
+        return if (fullPath.isNullOrEmpty()) {
+            SeparationContent(
+                packages = mutableListOf(
+                    PackageFilterInfo(
+                        packageKey = packageKey,
+                        versions = listOf(version!!)
+                    )
+                )
+            )
+        } else {
+            SeparationContent(
+                paths = mutableListOf(
+                    NodeFilterInfo(
+                        path = fullPath
+                    )
+                )
+            )
+        }
+    }
+
+    companion object {
+        private val logger = LoggerFactory.getLogger(SeparationRecoveryEventConsumer::class.java)
+    }
+}

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/pojo/RecoveryNodeInfo.kt

@@ -0,0 +1,35 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.separation.pojo
+
+data class RecoveryNodeInfo(
+    val projectId: String,
+    val repoName: String,
+    val fullPath: String,
+    val repoType: String,
+)