feat: 制品库支持深度归档 TencentBlueKing#1340 (TencentBlueKing#1432)

* feat: 制品库支持深度归档 TencentBlueKing#1340

* feat: 增加任务锁时长 TencentBlueKing#1340

* feat: 处理空任务和支持sha256并发归档 TencentBlueKing#1340

* feat: 控制归档中的任务并发数 TencentBlueKing#1340

* feat: 优化部分代码 TencentBlueKing#1340

* feat: 删除xz java TencentBlueKing#1340

* feat: 添加压缩指标 TencentBlueKing#1340

* feat: 归档任务增加文件大小选项 TencentBlueKing#1340

* feat: 可归档查询接口增加size参数 TencentBlueKing#1340

src/backend/archive/README.md

@@ -0,0 +1 @@
+# 归档服务

src/backend/archive/api-archive/build.gradle.kts

@@ -0,0 +1,6 @@
+dependencies {
+    api(project(":common:common-artifact:artifact-api"))
+    api(project(":common:common-query:query-api"))
+    api(project(":repository:api-repository"))
+    compileOnly("org.springframework.cloud:spring-cloud-openfeign-core")
+}

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/ArchiveFileNotFound.kt

@@ -0,0 +1,6 @@
+package com.tencent.bkrepo.archive
+
+import com.tencent.bkrepo.archive.constant.ArchiveMessageCode
+import com.tencent.bkrepo.common.api.exception.NotFoundException
+
+class ArchiveFileNotFound(file: String) : NotFoundException(ArchiveMessageCode.ARCHIVE_FILE_NOT_FOUND, file)

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/ArchiveStatus.kt

@@ -0,0 +1,48 @@
+package com.tencent.bkrepo.archive
+
+enum class ArchiveStatus {
+    /**
+     * 已创建
+     * */
+    CREATED,
+
+    /**
+     * 归档中
+     * */
+    ARCHIVING,
+
+    /**
+     * 已归档
+     * */
+    ARCHIVED,
+
+    /**
+     * 待恢复
+     * */
+    WAIT_TO_RESTORE,
+
+    /**
+     * 恢复中
+     * */
+    RESTORING,
+
+    /**
+     * 已恢复
+     * */
+    RESTORED,
+
+    /**
+     * 归档失败
+     * */
+    ARCHIVE_FAILED,
+
+    /**
+     * 恢复失败
+     * */
+    RESTORE_FAILED,
+
+    /**
+     * 已完成
+     * */
+    COMPLETED,
+}

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/api/ArchiveClient.kt

@@ -0,0 +1,36 @@
+package com.tencent.bkrepo.archive.api
+
+import com.tencent.bkrepo.archive.pojo.ArchiveFile
+import com.tencent.bkrepo.archive.request.ArchiveFileRequest
+import com.tencent.bkrepo.archive.request.CreateArchiveFileRequest
+import com.tencent.bkrepo.common.api.constant.ARCHIVE_SERVICE_NAME
+import com.tencent.bkrepo.common.api.pojo.Response
+import org.springframework.cloud.openfeign.FeignClient
+import org.springframework.web.bind.annotation.DeleteMapping
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.PostMapping
+import org.springframework.web.bind.annotation.RequestBody
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RequestParam
+
+@FeignClient(ARCHIVE_SERVICE_NAME)
+@RequestMapping("/service/archive")
+interface ArchiveClient {
+    @PostMapping("/archive")
+    fun archive(@RequestBody request: CreateArchiveFileRequest): Response<Void>
+
+    @DeleteMapping("/delete")
+    fun delete(@RequestBody request: ArchiveFileRequest): Response<Void>
+
+    @PostMapping("/restore")
+    fun restore(@RequestBody request: ArchiveFileRequest): Response<Void>
+
+    @PostMapping("/complete")
+    fun complete(@RequestBody request: ArchiveFileRequest): Response<Void>
+
+    @GetMapping
+    fun get(
+        @RequestParam sha256: String,
+        @RequestParam(required = false) storageCredentialsKey: String?,
+    ): Response<ArchiveFile?>
+}

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/constant/ArchiveConstants.kt

@@ -0,0 +1,4 @@
+package com.tencent.bkrepo.archive.constant
+
+const val XZ_SUFFIX = ".xz"
+const val DEEP_ARCHIVE = "DEEP_ARCHIVE"

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/constant/ArchiveMessageCode.kt

@@ -0,0 +1,13 @@
+package com.tencent.bkrepo.archive.constant
+
+import com.tencent.bkrepo.common.api.message.MessageCode
+
+enum class ArchiveMessageCode(private val key: String) : MessageCode {
+    ARCHIVE_FILE_NOT_FOUND("archive-file.not.found"),
+    RESTORE_COUNT_LIMIT("restore.count.limit"),
+    ;
+
+    override fun getBusinessCode() = ordinal + 1
+    override fun getKey() = key
+    override fun getModuleCode() = 25
+}

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/ArchiveFile.kt

@@ -0,0 +1,16 @@
+package com.tencent.bkrepo.archive.pojo
+
+import com.tencent.bkrepo.archive.ArchiveStatus
+import java.time.LocalDateTime
+
+data class ArchiveFile(
+    var id: String? = null,
+    var createdBy: String,
+    var createdDate: LocalDateTime,
+    var lastModifiedBy: String,
+    var lastModifiedDate: LocalDateTime,
+    val sha256: String,
+    val size: Long,
+    val storageCredentialsKey: String?,
+    var status: ArchiveStatus,
+)

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/ArchiveFileRequest.kt

@@ -0,0 +1,9 @@
+package com.tencent.bkrepo.archive.request
+
+import com.tencent.bkrepo.repository.constant.SYSTEM_USER
+
+data class ArchiveFileRequest(
+    val sha256: String,
+    val storageCredentialsKey: String?,
+    val operator: String = SYSTEM_USER,
+)

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/CreateArchiveFileRequest.kt

@@ -0,0 +1,10 @@
+package com.tencent.bkrepo.archive.request
+
+import com.tencent.bkrepo.repository.constant.SYSTEM_USER
+
+data class CreateArchiveFileRequest(
+    val sha256: String,
+    val size: Long,
+    val storageCredentialsKey: String?,
+    val operator: String = SYSTEM_USER,
+)

src/backend/archive/biz-archive/build.gradle.kts

@@ -0,0 +1,10 @@
+
+dependencies {
+    implementation(project(":archive:api-archive"))
+    api(project(":common:common-storage:storage-service"))
+    api(project(":common:common-security"))
+    api(project(":common:common-service"))
+    api(project(":common:common-mongo"))
+    api(project(":common:common-mongo-reactive"))
+    implementation("io.micrometer:micrometer-registry-prometheus")
+}

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/config/ArchiveConfiguration.kt

@@ -0,0 +1,8 @@
+package com.tencent.bkrepo.archive.config
+
+import org.springframework.boot.context.properties.EnableConfigurationProperties
+import org.springframework.context.annotation.Configuration
+
+@EnableConfigurationProperties(ArchiveProperties::class)
+@Configuration
+class ArchiveConfiguration

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/config/ArchiveProperties.kt

@@ -0,0 +1,50 @@
+package com.tencent.bkrepo.archive.config
+
+import com.tencent.bkrepo.common.storage.credentials.InnerCosCredentials
+import org.springframework.boot.context.properties.ConfigurationProperties
+import org.springframework.boot.context.properties.NestedConfigurationProperty
+import org.springframework.util.unit.DataSize
+
+@ConfigurationProperties("archive")
+data class ArchiveProperties(
+    @NestedConfigurationProperty
+    val cos: InnerCosCredentials = InnerCosCredentials(),
+    /**
+     * 工作路径
+     * */
+    var workDir: String = System.getProperty("java.io.tmpdir"),
+
+    /**
+     * 恢复出的临时副本的有效时长，单位为“天”
+     * */
+    var days: Int = 1,
+    /**
+     * 恢复模式
+     * */
+    var tier: String = "Standard",
+
+    var queryLimit: Int = 1000,
+
+    /**
+     * 磁盘可用空间阈值
+     * */
+    var threshold: DataSize = DataSize.ofMegabytes(10),
+    /**
+     * io thread num
+     * */
+    var ioThreads: Int = Runtime.getRuntime().availableProcessors(),
+    /**
+     * compress thread num
+     * */
+    var compressThreads: Int = 2,
+
+    /**
+     * xz memory limit
+     * */
+    var xzMemoryLimit: DataSize = DataSize.ofGigabytes(1),
+
+    /**
+     * 恢复数量限制
+     * */
+    var restoreLimit: Int = 1000,
+)

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/controller/service/ArchiveController.kt

@@ -0,0 +1,39 @@
+package com.tencent.bkrepo.archive.controller.service
+
+import com.tencent.bkrepo.archive.api.ArchiveClient
+import com.tencent.bkrepo.archive.pojo.ArchiveFile
+import com.tencent.bkrepo.archive.request.ArchiveFileRequest
+import com.tencent.bkrepo.archive.request.CreateArchiveFileRequest
+import com.tencent.bkrepo.archive.service.ArchiveService
+import com.tencent.bkrepo.common.api.pojo.Response
+import com.tencent.bkrepo.common.service.util.ResponseBuilder
+import org.springframework.web.bind.annotation.RestController
+
+@RestController
+class ArchiveController(
+    private val archiveService: ArchiveService,
+) : ArchiveClient {
+    override fun archive(request: CreateArchiveFileRequest): Response<Void> {
+        archiveService.archive(request)
+        return ResponseBuilder.success()
+    }
+
+    override fun delete(request: ArchiveFileRequest): Response<Void> {
+        archiveService.delete(request)
+        return ResponseBuilder.success()
+    }
+
+    override fun restore(request: ArchiveFileRequest): Response<Void> {
+        archiveService.restore(request)
+        return ResponseBuilder.success()
+    }
+
+    override fun get(sha256: String, storageCredentialsKey: String?): Response<ArchiveFile?> {
+        return ResponseBuilder.success(archiveService.get(sha256, storageCredentialsKey))
+    }
+
+    override fun complete(request: ArchiveFileRequest): Response<Void> {
+        archiveService.complete(request)
+        return ResponseBuilder.success()
+    }
+}

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/FileArchivedEvent.kt

@@ -0,0 +1,12 @@
+package com.tencent.bkrepo.archive.event
+
+import com.tencent.bkrepo.common.storage.monitor.Throughput
+
+/**
+ * 文件归档事件
+ * */
+data class FileArchivedEvent(
+    val sha256: String,
+    val storageCredentialsKey: String?,
+    val throughput: Throughput,
+)

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/FileCompressedEvent.kt

@@ -0,0 +1,13 @@
+package com.tencent.bkrepo.archive.event
+
+import com.tencent.bkrepo.common.storage.monitor.Throughput
+
+/**
+ * 文件归档事件
+ * */
+data class FileCompressedEvent(
+    val sha256: String,
+    val uncompressed: Long,
+    val compressed: Long,
+    val throughput: Throughput,
+)

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/FileRestoredEvent.kt

@@ -0,0 +1,12 @@
+package com.tencent.bkrepo.archive.event
+
+import com.tencent.bkrepo.common.storage.monitor.Throughput
+
+/**
+ * 文件恢复事件
+ * */
+data class FileRestoredEvent(
+    val sha256: String,
+    val storageCredentialsKey: String?,
+    val throughput: Throughput,
+)

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/extensions/ArciveExtensions.kt

@@ -0,0 +1,7 @@
+package com.tencent.bkrepo.archive.extensions
+
+import com.tencent.bkrepo.archive.model.TArchiveFile
+
+fun TArchiveFile.key(): String {
+    return "$sha256/$storageCredentialsKey"
+}

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/AbstractArchiveFileWrapperCallback.kt

@@ -0,0 +1,28 @@
+package com.tencent.bkrepo.archive.job
+
+import org.reactivestreams.Publisher
+import reactor.core.publisher.Mono
+
+/**
+ * 抽象的文件包装回调处理
+ *
+ * 负责一些通用处理，如异常处理
+ * */
+abstract class AbstractArchiveFileWrapperCallback : ArchiveFileWrapperCallback {
+    override fun onArchiveFileWrapper(fileWrapper: ArchiveFileWrapper): Publisher<ArchiveFileWrapper> {
+        // 如果上游发生错误，则直接返回
+        if (fileWrapper.throwable != null) {
+            return Mono.just(fileWrapper)
+        }
+        return process(fileWrapper).onErrorResume {
+            // 单个文件发生错误，将错误传递给下游，流继续执行。
+            fileWrapper.throwable = it
+            Mono.just(fileWrapper)
+        }
+    }
+
+    /**
+     * 处理文件
+     * */
+    abstract fun process(fileWrapper: ArchiveFileWrapper): Mono<ArchiveFileWrapper>
+}

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ArchiveFileWrapper.kt

@@ -0,0 +1,31 @@
+package com.tencent.bkrepo.archive.job
+
+import com.tencent.bkrepo.archive.model.TArchiveFile
+import java.nio.file.Path
+import java.time.LocalDateTime
+
+/**
+ * 归档文件包装器
+ * */
+data class ArchiveFileWrapper(
+    /**
+     * 待归档的文件
+     * */
+    val archiveFile: TArchiveFile,
+    /**
+     * 源文件路径
+     * */
+    var srcFilePath: Path? = null,
+    /**
+     * 压缩文件路径
+     * */
+    var compressedFilePath: Path? = null,
+    /**
+     * 错误信息
+     * */
+    var throwable: Throwable? = null,
+    /**
+     * 开始归档时间
+     * */
+    var startTime: LocalDateTime? = null,
+)

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ArchiveFileWrapperCallback.kt

@@ -0,0 +1,10 @@
+package com.tencent.bkrepo.archive.job
+
+import org.reactivestreams.Publisher
+
+/**
+ * 归档文件回调
+ * */
+interface ArchiveFileWrapperCallback {
+    fun onArchiveFileWrapper(fileWrapper: ArchiveFileWrapper): Publisher<ArchiveFileWrapper>
+}