From c603c54caaacc50ee59cb857a7e03fbd00788854 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 21 Aug 2024 00:39:46 +0100 Subject: [PATCH 1/5] towards production seqsets --- .github/workflows/backend-tests.yml | 4 ++-- .../org/loculus/backend/api/SeqSetCitations.kt | 2 +- .../SeqSetCitationsDatabaseService.kt | 2 +- .../loculus/templates/loculus-backend.yaml | 17 ++++++++--------- kubernetes/loculus/values.yaml | 6 ++---- 5 files changed, 14 insertions(+), 17 deletions(-) diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml index cb126e81b..7bed880bf 100644 --- a/.github/workflows/backend-tests.yml +++ b/.github/workflows/backend-tests.yml @@ -17,8 +17,8 @@ jobs: timeout-minutes: 15 env: CROSSREF_USERNAME: ${{ secrets.CROSSREF_USERNAME }} - CROSSREF_TEST_PASSWORD: ${{ secrets.CROSSREF_TEST_PASSWORD }} - CROSSREF_TEST_ENDPOINT: ${{ secrets.CROSSREF_TEST_ENDPOINT }} + CROSSREF_PASSWORD: ${{ secrets.CROSSREF_PASSWORD }} + CROSSREF_ENDPOINT: ${{ secrets.CROSSREF_ENDPOINT }} CROSSREF_DOI_PREFIX: ${{ secrets.CROSSREF_DOI_PREFIX }} steps: - name: Login to Docker Hub diff --git a/backend/src/main/kotlin/org/loculus/backend/api/SeqSetCitations.kt b/backend/src/main/kotlin/org/loculus/backend/api/SeqSetCitations.kt index d7ae4e55f..a556c7ee1 100644 --- a/backend/src/main/kotlin/org/loculus/backend/api/SeqSetCitations.kt +++ b/backend/src/main/kotlin/org/loculus/backend/api/SeqSetCitations.kt @@ -83,6 +83,6 @@ data class AuthorProfile( ) object SeqSetCitationsConstants { - const val DOI_PREFIX = "placeholder" + const val DOI_PREFIX = "placeholder" // I think we can delete this? const val DOI_WEEKLY_RATE_LIMIT = 7 } diff --git a/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt index 6c609c991..9130d72e5 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt @@ -333,7 +333,7 @@ class SeqSetCitationsDatabaseService( validateCreateSeqSetDOI(username, seqSetId, version) - val seqSetDOI = "${SeqSetCitationsConstants.DOI_PREFIX}/$seqSetId.$version" + val seqSetDOI = "${SeqSetCitationsConstants.DOI_PREFIX}/$seqSetId.$version" // I think we need to get this to not use the constant but the set value SeqSetsTable.update( { diff --git a/kubernetes/loculus/templates/loculus-backend.yaml b/kubernetes/loculus/templates/loculus-backend.yaml index 7ce71aa50..f2494283a 100644 --- a/kubernetes/loculus/templates/loculus-backend.yaml +++ b/kubernetes/loculus/templates/loculus-backend.yaml @@ -46,9 +46,9 @@ spec: - containerPort: 8079 args: - "--crossref.doi-prefix=$(CROSSREF_DOI_PREFIX)" - - "--crossref.endpoint=$(CROSSREF_TEST_ENDPOINT)" + - "--crossref.endpoint=$(CROSSREF_ENDPOINT)" - "--crossref.username=$(CROSSREF_USERNAME)" - - "--crossref.password=$(CROSSREF_TEST_PASSWORD)" + - "--crossref.password=$(CROSSREF_PASSWORD)" - "--keycloak.password=$(BACKEND_KEYCLOAK_PASSWORD)" - "--keycloak.realm=loculus" - "--keycloak.client=backend-client" @@ -71,16 +71,15 @@ spec: secretKeyRef: name: crossref key: username - - name: CROSSREF_TEST_PASSWORD + - name: CROSSREF_PASSWORD valueFrom: secretKeyRef: name: crossref - key: testPassword - - name: CROSSREF_LIVE_PASSWORD - valueFrom: - secretKeyRef: - name: crossref - key: livePassword + key: password + - name: CROSSREF_DOI_PREFIX + value: {{$.Values.crossref.DOIPrefix}} + - name: CROSSREF_ENDPOINT + value: {{$.Values.crossref.endpoint }} {{- end }} - name: BACKEND_KEYCLOAK_PASSWORD valueFrom: diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index 1d12e1be7..1a48a0e47 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -1,8 +1,7 @@ environment: server crossRef: DOIPrefix: "10.62599" - testEndpoint: "https://test.crossref.org" - liveEndpoint: "https://doi.crossref.org" + endpoint: "https://test.crossref.org" disableWebsite: false disableBackend: false disablePreprocessing: false @@ -1481,8 +1480,7 @@ secrets: type: raw data: username: "dummy" - testPassword: "dummy" - livePassword: "dummy" + password: "dummy" service-accounts: type: autogen data: From b89e12d49104ddc284f31fa562e6e6c838a90eb9 Mon Sep 17 00:00:00 2001 From: Chaoran Chen Date: Wed, 21 Aug 2024 06:57:32 +0200 Subject: [PATCH 2/5] adapt kotlin code --- .../org/loculus/backend/api/SeqSetCitations.kt | 1 - .../backend/service/crossref/CrossRefService.kt | 14 ++++++++++++-- .../SeqSetCitationsDatabaseService.kt | 5 ++++- backend/src/test/resources/application.properties | 1 + 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/backend/src/main/kotlin/org/loculus/backend/api/SeqSetCitations.kt b/backend/src/main/kotlin/org/loculus/backend/api/SeqSetCitations.kt index a556c7ee1..84b0ce97f 100644 --- a/backend/src/main/kotlin/org/loculus/backend/api/SeqSetCitations.kt +++ b/backend/src/main/kotlin/org/loculus/backend/api/SeqSetCitations.kt @@ -83,6 +83,5 @@ data class AuthorProfile( ) object SeqSetCitationsConstants { - const val DOI_PREFIX = "placeholder" // I think we can delete this? const val DOI_WEEKLY_RATE_LIMIT = 7 } diff --git a/backend/src/main/kotlin/org/loculus/backend/service/crossref/CrossRefService.kt b/backend/src/main/kotlin/org/loculus/backend/service/crossref/CrossRefService.kt index b6461a9e7..ecffdd362 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/crossref/CrossRefService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/crossref/CrossRefService.kt @@ -20,17 +20,27 @@ import java.util.UUID private val log = KotlinLogging.logger { } @ConfigurationProperties(prefix = "crossref") -data class CrossRefServiceProperties(val endpoint: String?, val username: String?, val password: String?) +data class CrossRefServiceProperties( + val endpoint: String?, + val username: String?, + val password: String?, + val doiPrefix: String?, +) @Service class CrossRefService(private val crossRefServiceProperties: CrossRefServiceProperties) { val isActive = crossRefServiceProperties.endpoint != null && crossRefServiceProperties.username != null && - crossRefServiceProperties.password != null + crossRefServiceProperties.password != null && + crossRefServiceProperties.doiPrefix != null val dateTimeFormatterMM = DateTimeFormatter.ofPattern("MM") val dateTimeFormatterdd = DateTimeFormatter.ofPattern("dd") val dateTimeFormatteryyyy = DateTimeFormatter.ofPattern("yyyy") + init { + println("heeelo2 $crossRefServiceProperties") + } + private fun checkIsActive() { if (!isActive) { throw RuntimeException("The CrossRefService is not active as it has not been configured.") diff --git a/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt index 9130d72e5..753a451a5 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt @@ -32,6 +32,7 @@ import org.loculus.backend.auth.AuthenticatedUser import org.loculus.backend.config.BackendConfig import org.loculus.backend.controller.NotFoundException import org.loculus.backend.controller.UnprocessableEntityException +import org.loculus.backend.service.crossref.CrossRefServiceProperties import org.loculus.backend.service.submission.AccessionPreconditionValidator import org.loculus.backend.utils.getNextSequenceNumber import org.springframework.stereotype.Service @@ -47,6 +48,7 @@ private val log = KotlinLogging.logger { } class SeqSetCitationsDatabaseService( private val accessionPreconditionValidator: AccessionPreconditionValidator, private val backendConfig: BackendConfig, + private val crossRefServiceProperties: CrossRefServiceProperties, pool: DataSource, ) { init { @@ -333,7 +335,8 @@ class SeqSetCitationsDatabaseService( validateCreateSeqSetDOI(username, seqSetId, version) - val seqSetDOI = "${SeqSetCitationsConstants.DOI_PREFIX}/$seqSetId.$version" // I think we need to get this to not use the constant but the set value + val doiPrefix = crossRefServiceProperties.doiPrefix ?: "no-prefix-configured" + val seqSetDOI = "$doiPrefix/$seqSetId.$version" SeqSetsTable.update( { diff --git a/backend/src/test/resources/application.properties b/backend/src/test/resources/application.properties index 77e5aafde..23580ef78 100644 --- a/backend/src/test/resources/application.properties +++ b/backend/src/test/resources/application.properties @@ -4,6 +4,7 @@ loculus.config.path=src/test/resources/backend_config.json crossref.endpoint=dummy crossref.username=dummy crossref.password=dummy +crossref.doi-prefix=placeholder keycloak.user=dummy keycloak.password=dummy From c570fcd23dd48234eff4b22580e4e19f83f567f7 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 21 Aug 2024 14:05:55 +0100 Subject: [PATCH 3/5] Update backend/src/main/kotlin/org/loculus/backend/service/crossref/CrossRefService.kt Co-authored-by: Chaoran Chen --- .../org/loculus/backend/service/crossref/CrossRefService.kt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/backend/src/main/kotlin/org/loculus/backend/service/crossref/CrossRefService.kt b/backend/src/main/kotlin/org/loculus/backend/service/crossref/CrossRefService.kt index ecffdd362..c825d02e2 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/crossref/CrossRefService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/crossref/CrossRefService.kt @@ -37,10 +37,6 @@ class CrossRefService(private val crossRefServiceProperties: CrossRefServiceProp val dateTimeFormatterdd = DateTimeFormatter.ofPattern("dd") val dateTimeFormatteryyyy = DateTimeFormatter.ofPattern("yyyy") - init { - println("heeelo2 $crossRefServiceProperties") - } - private fun checkIsActive() { if (!isActive) { throw RuntimeException("The CrossRefService is not active as it has not been configured.") From 53ef6524c10f905173d69573137eb560116436a8 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 21 Aug 2024 14:12:38 +0100 Subject: [PATCH 4/5] update --- kubernetes/loculus/templates/loculus-backend.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/loculus/templates/loculus-backend.yaml b/kubernetes/loculus/templates/loculus-backend.yaml index f2494283a..8b9e2f4aa 100644 --- a/kubernetes/loculus/templates/loculus-backend.yaml +++ b/kubernetes/loculus/templates/loculus-backend.yaml @@ -77,9 +77,9 @@ spec: name: crossref key: password - name: CROSSREF_DOI_PREFIX - value: {{$.Values.crossref.DOIPrefix}} + value: {{$.Values.crossRef.DOIPrefix }} - name: CROSSREF_ENDPOINT - value: {{$.Values.crossref.endpoint }} + value: {{$.Values.crossRef.endpoint }} {{- end }} - name: BACKEND_KEYCLOAK_PASSWORD valueFrom: From 0f8b817b6cd2a75633f7ed36db09b983ff107a7d Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 21 Aug 2024 14:19:41 +0100 Subject: [PATCH 5/5] add quote --- kubernetes/loculus/templates/loculus-backend.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/loculus/templates/loculus-backend.yaml b/kubernetes/loculus/templates/loculus-backend.yaml index 8b9e2f4aa..14692406c 100644 --- a/kubernetes/loculus/templates/loculus-backend.yaml +++ b/kubernetes/loculus/templates/loculus-backend.yaml @@ -77,9 +77,9 @@ spec: name: crossref key: password - name: CROSSREF_DOI_PREFIX - value: {{$.Values.crossRef.DOIPrefix }} + value: {{$.Values.crossRef.DOIPrefix | quote }} - name: CROSSREF_ENDPOINT - value: {{$.Values.crossRef.endpoint }} + value: {{$.Values.crossRef.endpoint | quote }} {{- end }} - name: BACKEND_KEYCLOAK_PASSWORD valueFrom: