Skip to content

Commit

Permalink
feat(website,backend): adapt endpoint, and frontend, add tests, add c…
Browse files Browse the repository at this point in the history
…ontrains
  • Loading branch information
TobiasKampmann committed Jan 19, 2024
1 parent 53c5701 commit 4f72903
Show file tree
Hide file tree
Showing 27 changed files with 555 additions and 85 deletions.
109 changes: 109 additions & 0 deletions backend/src/main/kotlin/org/loculus/backend/api/DataUseTerms.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
package org.loculus.backend.api

import com.fasterxml.jackson.annotation.JsonIgnore
import com.fasterxml.jackson.annotation.JsonPropertyOrder
import com.fasterxml.jackson.annotation.JsonSubTypes
import com.fasterxml.jackson.annotation.JsonTypeInfo
import com.fasterxml.jackson.annotation.JsonTypeName
import com.fasterxml.jackson.core.JsonGenerator
import com.fasterxml.jackson.databind.SerializerProvider
import com.fasterxml.jackson.databind.annotation.JsonSerialize
import com.fasterxml.jackson.databind.ser.std.StdSerializer
import kotlinx.datetime.Clock
import kotlinx.datetime.DateTimeUnit.Companion.YEAR
import kotlinx.datetime.LocalDate
import kotlinx.datetime.LocalDateTime
import kotlinx.datetime.TimeZone
import kotlinx.datetime.plus
import kotlinx.datetime.toLocalDateTime
import mu.KotlinLogging
import org.loculus.backend.config.logger
import org.loculus.backend.controller.BadRequestException

enum class DataUseTermsType {
OPEN,
RESTRICTED,
}

val logger = KotlinLogging.logger { }

@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
@JsonSubTypes(
JsonSubTypes.Type(value = DataUseTerms.Open::class, name = "OPEN"),
JsonSubTypes.Type(value = DataUseTerms.Restricted::class, name = "RESTRICTED"),
)
@JsonPropertyOrder(value = ["type", "restrictedUntil", "changeDateTime"])
sealed interface DataUseTerms {
val type: DataUseTermsType

@JsonTypeName("OPEN")
data class Open(private val dummy: String = "") :
DataUseTerms {
@JsonIgnore
override val type = DataUseTermsType.OPEN
}

@JsonTypeName("RESTRICTED")
data class Restricted(
@JsonSerialize(using = LocalDateSerializer::class)
val restrictedUntil: LocalDate,
) : DataUseTerms {
@JsonIgnore
override val type = DataUseTermsType.RESTRICTED
}

companion object {
fun fromParameters(type: DataUseTermsType, restrictedUntilString: String?): DataUseTerms {
logger.info { "Creating DataUseTerms from parameters: type=$type, restrictedUntil=$restrictedUntilString" }
return when (type) {
DataUseTermsType.OPEN -> Open()
DataUseTermsType.RESTRICTED -> {
val restrictedUntil = parseRestrictedUntil(restrictedUntilString)
validateRestrictedUntil(restrictedUntil)
Restricted(restrictedUntil)
}
}
}

private fun parseRestrictedUntil(restrictedUntilString: String?): LocalDate {
if (restrictedUntilString == null) {
throw BadRequestException("The date 'restrictedUntil' must be set if 'dataUseTermsType' is RESTRICTED.")
}
return try {
LocalDate.parse(restrictedUntilString)
} catch (e: Exception) {
throw BadRequestException(
"The date 'restrictedUntil' must be a valid date in the format YYYY-MM-DD: $restrictedUntilString.",
)
}
}

private fun validateRestrictedUntil(restrictedUntil: LocalDate) {
val now = Clock.System.now().toLocalDateTime(TimeZone.UTC).date
val oneYearFromNow = now.plus(1, YEAR)

if (restrictedUntil < now) {
throw BadRequestException(
"The date 'restrictedUntil' must be in the future, up to a maximum of 1 year from now.",
)
}
if (restrictedUntil > oneYearFromNow) {
throw BadRequestException(
"The date 'restrictedUntil' must not exceed 1 year from today.",
)
}
}
}
}

class LocalDateSerializer : StdSerializer<LocalDate>(LocalDate::class.java) {
override fun serialize(value: LocalDate, gen: JsonGenerator, provider: SerializerProvider) {
gen.writeString(value.toString())
}
}

class LocalDateTimeSerializer : StdSerializer<LocalDateTime>(LocalDateTime::class.java) {
override fun serialize(value: LocalDateTime, gen: JsonGenerator, provider: SerializerProvider) {
gen.writeString(value.toString())
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ data class SubmissionIdMapping(
override val accession: Accession,
override val version: Version,
val submissionId: String,
val dataUseTerms: DataUseTerms? = null,
) : AccessionVersionInterface

fun <T : AccessionVersionInterface> List<T>.toPairs() = map { Pair(it.accession, it.version) }
Expand Down Expand Up @@ -148,12 +149,6 @@ data class SequenceEntryStatus(
val isRevocation: Boolean = false,
) : AccessionVersionInterface

data class RevisedData(
val submissionId: String,
val accession: Accession,
val originalData: OriginalData,
)

data class UnprocessedData(
@Schema(example = "123") override val accession: Accession,
@Schema(example = "1") override val version: Version,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@ package org.loculus.backend.controller
import io.swagger.v3.oas.annotations.Operation
import io.swagger.v3.oas.annotations.Parameter
import io.swagger.v3.oas.annotations.security.SecurityRequirement
import org.loculus.backend.service.datauseterms.DataUseTerms
import org.loculus.backend.api.DataUseTerms
import org.loculus.backend.service.datauseterms.DataUseTermsDatabaseService
import org.loculus.backend.utils.Accession
import org.springframework.http.HttpStatus
import org.springframework.http.MediaType
import org.springframework.web.bind.annotation.PutMapping
import org.springframework.web.bind.annotation.RequestBody
import org.springframework.web.bind.annotation.RequestParam
import org.springframework.web.bind.annotation.ResponseStatus
import org.springframework.web.bind.annotation.RestController
Expand All @@ -27,9 +28,12 @@ class DataUseTermsController(
@Parameter(
description = "The accession of the dataset to set the data use terms for",
) @RequestParam accession: Accession,
@Parameter(
description = "The new data use terms",
) @RequestBody newDataUseTerms: DataUseTerms,
) = dataUseTermsDatabaseService.setNewDataUseTerms(
accession,
listOf(accession),
username,
DataUseTerms(),
DataUseTerms.Open(),
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import jakarta.validation.Valid
import jakarta.validation.constraints.Max
import mu.KotlinLogging
import org.loculus.backend.api.AccessionVersion
import org.loculus.backend.api.DataUseTerms
import org.loculus.backend.api.DataUseTermsType
import org.loculus.backend.api.Organism
import org.loculus.backend.api.ProcessedData
import org.loculus.backend.api.SequenceEntryStatus
Expand Down Expand Up @@ -67,13 +69,22 @@ class SubmissionController(
@Parameter(description = GROUP_DESCRIPTION) @RequestParam groupName: String,
@Parameter(description = METADATA_FILE_DESCRIPTION) @RequestParam metadataFile: MultipartFile,
@Parameter(description = SEQUENCE_FILE_DESCRIPTION) @RequestParam sequenceFile: MultipartFile,
@Parameter(description = "Data Use terms under which data is released.")
@RequestParam
dataUseTermsType: DataUseTermsType,
@Parameter(
description = "Mandatory when data use terms are set to 'RESTRICTED'." +
" It is the date when the sequence entries will become 'OPEN'." +
" Format: YYYY-MM-DD",
) @RequestParam restrictedUntil: String?,
): List<SubmissionIdMapping> {
val params = SubmissionParams.OriginalSubmissionParams(
organism,
username,
metadataFile,
sequenceFile,
groupName,
DataUseTerms.fromParameters(dataUseTermsType, restrictedUntil),
)
return submitModel.processSubmissions(UUID.randomUUID().toString(), params)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import mu.KotlinLogging
import org.apache.commons.compress.archivers.zip.ZipFile
import org.apache.commons.compress.compressors.CompressorStreamFactory
import org.jetbrains.exposed.exceptions.ExposedSQLException
import org.loculus.backend.api.DataUseTerms
import org.loculus.backend.api.Organism
import org.loculus.backend.api.SubmissionIdMapping
import org.loculus.backend.controller.BadRequestException
Expand Down Expand Up @@ -46,6 +47,7 @@ interface SubmissionParams {
override val metadataFile: MultipartFile,
override val sequenceFile: MultipartFile,
val groupName: String,
val dataUseTerms: DataUseTerms,
) : SubmissionParams {
override val uploadType: UploadType = UploadType.ORIGINAL
}
Expand Down Expand Up @@ -116,7 +118,7 @@ class SubmitModel(
}

log.debug { "Persisting submission with uploadId $uploadId" }
uploadDatabaseService.mapAndCopy(uploadId, submissionParams.uploadType)
uploadDatabaseService.mapAndCopy(uploadId, submissionParams)
} finally {
uploadDatabaseService.deleteUploadData(uploadId)
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,50 +1,42 @@
package org.loculus.backend.service.datauseterms

import kotlinx.datetime.Clock
import kotlinx.datetime.LocalDateTime
import kotlinx.datetime.TimeZone
import kotlinx.datetime.toLocalDateTime
import mu.KotlinLogging
import org.jetbrains.exposed.exceptions.ExposedSQLException
import org.jetbrains.exposed.sql.insert
import org.jetbrains.exposed.sql.batchInsert
import org.loculus.backend.api.DataUseTerms
import org.springframework.stereotype.Service
import org.springframework.transaction.annotation.Transactional

enum class DataUseTermsType {
RESTRICTED,
OPEN,
}

data class DataUseTerms(
val restrictedUntil: LocalDateTime? = null,
val changeDateTime: LocalDateTime = Clock.System.now().toLocalDateTime(TimeZone.UTC),
val dataUseTermsType: DataUseTermsType = DataUseTermsType.OPEN,
)

private val log = KotlinLogging.logger { }

@Service
@Transactional
class DataUseTermsDatabaseService {

fun setNewDataUseTerms(accession: String, username: String, newDataUseTerms: DataUseTerms) {
fun setNewDataUseTerms(accessions: List<String>, username: String, newDataUseTerms: DataUseTerms) {
log.info {
"Setting new data use terms for accession $accession. " +
"Setting new data use terms for accessions $accessions. " +
"Just an entry in the new Table. " +
"Will be filled with real juicy logic in the next tickets. See #760 ff. "
}
val now = Clock.System.now().toLocalDateTime(TimeZone.UTC)
try {
DataUseTermsTable.insert {
it[accessionColumn] = accession
it[changeDateColumn] = now
it[dataUseTermsTypeColumn] = newDataUseTerms.dataUseTermsType
it[restrictedUntilColumn] = newDataUseTerms.restrictedUntil
it[userNameColumn] = username

DataUseTermsTable.batchInsert(accessions) {
this[DataUseTermsTable.accessionColumn] = it
this[DataUseTermsTable.changeDateColumn] = now
this[DataUseTermsTable.dataUseTermsTypeColumn] = newDataUseTerms.type
this[DataUseTermsTable.restrictedUntilColumn] = when (newDataUseTerms) {
is DataUseTerms.Restricted -> {
newDataUseTerms.restrictedUntil
}

else -> {
null
}
}
} catch (e: ExposedSQLException) {
log.info("Error: ${e.sqlState}")
throw e
this[DataUseTermsTable.userNameColumn] = username
}
}
}
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
package org.loculus.backend.service.datauseterms

import org.jetbrains.exposed.sql.Table
import org.jetbrains.exposed.sql.kotlin.datetime.date
import org.jetbrains.exposed.sql.kotlin.datetime.datetime
import org.loculus.backend.api.DataUseTermsType

const val DATA_USE_TERMS_TABLE_NAME = "data_use_terms_table"

object DataUseTermsTable : Table(DATA_USE_TERMS_TABLE_NAME) {
val accessionColumn = text("accession")
val changeDateColumn = datetime("change_date")
val dataUseTermsTypeColumn = enumeration("data_use_terms_type", DataUseTermsType::class)
val restrictedUntilColumn = datetime("restricted_until").nullable()
val restrictedUntilColumn = date("restricted_until").nullable()
val userNameColumn = text("user_name")
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ import org.loculus.backend.api.Organism
import org.loculus.backend.api.Status
import org.loculus.backend.api.SubmissionIdMapping
import org.loculus.backend.model.SubmissionId
import org.loculus.backend.model.SubmissionParams
import org.loculus.backend.model.UploadType
import org.loculus.backend.service.datauseterms.DataUseTermsDatabaseService
import org.loculus.backend.service.submission.MetadataUploadAuxTable.accessionColumn
import org.loculus.backend.service.submission.MetadataUploadAuxTable.groupNameColumn
import org.loculus.backend.service.submission.MetadataUploadAuxTable.metadataColumn
Expand Down Expand Up @@ -43,6 +45,7 @@ class UploadDatabaseService(
private val parseFastaHeader: ParseFastaHeader,
private val compressor: CompressionService,
private val submissionPreconditionValidator: SubmissionPreconditionValidator,
private val dataUseTermsDatabaseService: DataUseTermsDatabaseService,
) {

fun batchInsertMetadataInAuxTable(
Expand Down Expand Up @@ -112,11 +115,13 @@ class UploadDatabaseService(
},
)

fun mapAndCopy(uploadId: String, uploadType: UploadType): List<SubmissionIdMapping> = transaction {
log.debug { "mapping and copying sequences with UploadId $uploadId and uploadType: $uploadType" }
fun mapAndCopy(uploadId: String, submissionParams: SubmissionParams): List<SubmissionIdMapping> = transaction {
log.debug {
"mapping and copying sequences with UploadId $uploadId and uploadType: $submissionParams.uploadType"
}

exec(
generateMapAndCopyStatement(uploadType),
val insertionResult = exec(
generateMapAndCopyStatement(submissionParams.uploadType),
listOf(
Pair(VarCharColumnType(), uploadId),
),
Expand All @@ -132,6 +137,27 @@ class UploadDatabaseService(
}
result.toList()
} ?: emptyList()

val result = if (submissionParams is SubmissionParams.OriginalSubmissionParams) {
dataUseTermsDatabaseService.setNewDataUseTerms(
insertionResult.map { it.accession },
submissionParams.username,
submissionParams.dataUseTerms,
)

insertionResult.map {
SubmissionIdMapping(
it.accession,
it.version,
it.submissionId,
submissionParams.dataUseTerms,
)
}
} else {
insertionResult
}

return@transaction result
}

fun deleteUploadData(uploadId: String) {
Expand Down
Loading

0 comments on commit 4f72903

Please sign in to comment.