feat(backend, website): add pagination

backend/src/main/kotlin/org/loculus/backend/api/SubmissionTypes.kt

@@ -146,6 +146,11 @@ enum class PreprocessingAnnotationSourceType {
     NucleotideSequence,
 }
 
+data class GetSequenceResponse(
+    val sequenceEntries: List<SequenceEntryStatus>,
+    val statusCounts: Map<Status, Int>,
+)
+
 data class SequenceEntryStatus(
     override val accession: Accession,
     override val version: Version,

backend/src/main/kotlin/org/loculus/backend/controller/DatasetCitationsController.kt

@@ -92,7 +92,7 @@ class DatasetCitationsController(
     fun getUserCitedByDataset(@UsernameFromJwt username: String): CitedBy {
         val statusFilter = listOf(APPROVED_FOR_RELEASE)
         val userSequences = submissionDatabaseService.getSequences(username, null, null, statusFilter)
-        return datasetCitationsService.getUserCitedByDataset(userSequences)
+        return datasetCitationsService.getUserCitedByDataset(userSequences.sequenceEntries)
     }
 
     @Operation(description = "Get count of dataset cited by publications")

backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt

@@ -14,9 +14,9 @@ import org.loculus.backend.api.AccessionVersion
 import org.loculus.backend.api.Accessions
 import org.loculus.backend.api.DataUseTerms
 import org.loculus.backend.api.DataUseTermsType
+import org.loculus.backend.api.GetSequenceResponse
 import org.loculus.backend.api.Organism
 import org.loculus.backend.api.ProcessedData
-import org.loculus.backend.api.SequenceEntryStatus
 import org.loculus.backend.api.SequenceEntryVersionToEdit
 import org.loculus.backend.api.Status
 import org.loculus.backend.api.SubmissionIdMapping
@@ -252,8 +252,20 @@ class SubmissionController(
         @RequestParam(required = false)
         statusesFilter: List<Status>?,
         @UsernameFromJwt username: String,
-    ): List<SequenceEntryStatus> =
-        submissionDatabaseService.getSequences(username, organism, groupsFilter, statusesFilter)
+        @Parameter(
+            description = "Part of pagination parameters. Page number starts from 0. " +
+                "If page or size are not provided, all sequences are returned.",
+        )
+        @RequestParam(required = false)
+        page: Int?,
+        @Parameter(
+            description = "Part of pagination parameters. Number of sequences per page. " +
+                "If page or size are not provided, all sequences are returned.",
+        )
+        @RequestParam(required = false)
+        size: Int?,
+    ): GetSequenceResponse =
+        submissionDatabaseService.getSequences(username, organism, groupsFilter, statusesFilter, page, size)
 
     @Operation(description = APPROVE_PROCESSED_DATA_DESCRIPTION)
     @ResponseStatus(HttpStatus.NO_CONTENT)

backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt

@@ -26,6 +26,7 @@ import org.loculus.backend.api.AccessionVersion
 import org.loculus.backend.api.AccessionVersionInterface
 import org.loculus.backend.api.DataUseTerms
 import org.loculus.backend.api.DataUseTermsType
+import org.loculus.backend.api.GetSequenceResponse
 import org.loculus.backend.api.Group
 import org.loculus.backend.api.Organism
 import org.loculus.backend.api.ProcessedData
@@ -363,10 +364,15 @@ class SubmissionDatabaseService(
         organism: Organism?,
         groupsFilter: List<String>?,
         statusesFilter: List<Status>?,
-    ): List<SequenceEntryStatus> {
-        log.info { "getting sequence for user $username (groupFilter: $groupsFilter in statuses $statusesFilter" }
+        page: Int? = null,
+        size: Int? = null,
+    ): GetSequenceResponse {
+        log.info {
+            "getting sequence for user $username (groupFilter: $groupsFilter in statuses $statusesFilter)." +
+                " Page $page of size $size "
+        }
 
-        val validatedGroupNames = if (groupsFilter === null) {
+        val validatedGroupNames = if (groupsFilter == null) {
             groupManagementDatabaseService.getGroupsOfUser(username)
         } else {
             groupManagementPreconditionValidator.validateUserInExistingGroups(groupsFilter, username)
@@ -377,10 +383,14 @@ class SubmissionDatabaseService(
 
         sequenceEntriesTableProvider.get(organism).let { table ->
             val query = table
-                .join(DataUseTermsTable, JoinType.LEFT, additionalConstraint = {
-                    (table.accessionColumn eq DataUseTermsTable.accessionColumn) and
-                        (DataUseTermsTable.isNewestDataUseTerms)
-                })
+                .join(
+                    DataUseTermsTable,
+                    JoinType.LEFT,
+                    additionalConstraint = {
+                        (table.accessionColumn eq DataUseTermsTable.accessionColumn) and
+                            (DataUseTermsTable.isNewestDataUseTerms)
+                    },
+                )
                 .slice(
                     table.accessionColumn,
                     table.versionColumn,
@@ -399,27 +409,40 @@ class SubmissionDatabaseService(
                             table.groupIsOneOf(validatedGroupNames)
                     },
                 )
+                .orderBy(table.accessionColumn)
 
             if (organism != null) {
                 query.andWhere { table.organismIs(organism) }
             }
 
-            return query
-                .sortedBy { it[table.submittedAtColumn] }
-                .map { row ->
-                    SequenceEntryStatus(
-                        row[table.accessionColumn],
-                        row[table.versionColumn],
-                        Status.fromString(row[table.statusColumn]),
-                        row[table.groupNameColumn],
-                        row[table.isRevocationColumn],
-                        row[table.submissionIdColumn],
-                        dataUseTerms = DataUseTerms.fromParameters(
-                            DataUseTermsType.fromString(row[DataUseTermsTable.dataUseTermsTypeColumn]),
-                            row[DataUseTermsTable.restrictedUntilColumn],
-                        ),
-                    )
-                }
+            val statusCounts: Map<Status, Int> = listOfStatuses.associateWith { status ->
+                query.count { it[table.statusColumn] == status.name }
+            }
+
+            val pagedQuery = if (page != null && size != null) {
+                query.limit(size, (page * size).toLong())
+            } else {
+                query
+            }
+
+            return GetSequenceResponse(
+                sequenceEntries = pagedQuery
+                    .map { row ->
+                        SequenceEntryStatus(
+                            row[table.accessionColumn],
+                            row[table.versionColumn],
+                            Status.fromString(row[table.statusColumn]),
+                            row[table.groupNameColumn],
+                            row[table.isRevocationColumn],
+                            row[table.submissionIdColumn],
+                            dataUseTerms = DataUseTerms.fromParameters(
+                                DataUseTermsType.fromString(row[DataUseTermsTable.dataUseTermsTypeColumn]),
+                                row[DataUseTermsTable.restrictedUntilColumn],
+                            ),
+                        )
+                    },
+                statusCounts = statusCounts,
+            )
         }
     }
 

backend/src/test/kotlin/org/loculus/backend/controller/datasetcitations/CitationEndpointsTest.kt

@@ -7,6 +7,7 @@ import org.junit.jupiter.api.Test
 import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.MethodSource
 import org.loculus.backend.api.DataUseTerms
+import org.loculus.backend.api.GetSequenceResponse
 import org.loculus.backend.api.SequenceEntryStatus
 import org.loculus.backend.api.Status
 import org.loculus.backend.controller.EndpointTest
@@ -37,7 +38,9 @@ class CitationEndpointsTest(
 
     @Test
     fun `WHEN calling get user cited by dataset of non-existing user THEN returns empty results`() {
-        every { submissionDatabaseService.getSequences(any(), any(), any(), any()) } returns emptyList()
+        every {
+            submissionDatabaseService.getSequences(any(), any(), any(), any())
+        } returns GetSequenceResponse(sequenceEntries = emptyList(), statusCounts = emptyMap())
 
         client.getUserCitedByDataset()
             .andExpect(status().isOk)
@@ -61,16 +64,21 @@ class CitationEndpointsTest(
 
     @Test
     fun `WHEN calling get dataset cited by publication of existing dataset THEN returns results`() {
-        every { submissionDatabaseService.getSequences(any(), any(), any(), any()) } returns listOf(
-            SequenceEntryStatus(
-                "mock-sequence-accession",
-                1L,
-                Status.APPROVED_FOR_RELEASE,
-                "mock-group",
-                false,
-                "mock-submission-id",
-                DataUseTerms.Open,
+        every {
+            submissionDatabaseService.getSequences(any(), any(), any(), any())
+        } returns GetSequenceResponse(
+            sequenceEntries = listOf(
+                SequenceEntryStatus(
+                    "mock-sequence-accession",
+                    1L,
+                    Status.APPROVED_FOR_RELEASE,
+                    "mock-group",
+                    false,
+                    "mock-submission-id",
+                    DataUseTerms.Open,
+                ),
             ),
+            statusCounts = mapOf(Status.APPROVED_FOR_RELEASE to 1),
         )
 
         val datasetResult = client.createDataset()

backend/src/test/kotlin/org/loculus/backend/controller/submission/GetSequencesEndpointTest.kt

@@ -11,8 +11,12 @@ import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.MethodSource
 import org.loculus.backend.api.AccessionVersion
 import org.loculus.backend.api.Status
+import org.loculus.backend.api.Status.APPROVED_FOR_RELEASE
 import org.loculus.backend.api.Status.AWAITING_APPROVAL
+import org.loculus.backend.api.Status.AWAITING_APPROVAL_FOR_REVOCATION
+import org.loculus.backend.api.Status.HAS_ERRORS
 import org.loculus.backend.api.Status.IN_PROCESSING
+import org.loculus.backend.api.Status.RECEIVED
 import org.loculus.backend.controller.ALTERNATIVE_DEFAULT_GROUP_NAME
 import org.loculus.backend.controller.ALTERNATIVE_DEFAULT_USER_NAME
 import org.loculus.backend.controller.DEFAULT_GROUP_NAME
@@ -45,13 +49,13 @@ class GetSequencesEndpointTest(
 
         val sequencesOfUser = convenienceClient.getSequenceEntries(
             username = DEFAULT_USER_NAME,
-        )
+        ).sequenceEntries
 
         assertThat(sequencesOfUser, hasSize(NUMBER_OF_SEQUENCES))
 
         val sequencesOfAlternativeUser = convenienceClient.getSequenceEntries(
             username = ALTERNATIVE_DEFAULT_USER_NAME,
-        )
+        ).sequenceEntries
 
         assertThat(sequencesOfAlternativeUser, hasSize(NUMBER_OF_SEQUENCES))
     }
@@ -63,7 +67,7 @@ class GetSequencesEndpointTest(
         val sequencesOfUser = convenienceClient.getSequenceEntries(
             username = DEFAULT_USER_NAME,
             groupsFilter = listOf(ALTERNATIVE_DEFAULT_GROUP_NAME),
-        )
+        ).sequenceEntries
 
         assertThat(sequencesOfUser, hasSize(0))
     }
@@ -89,7 +93,7 @@ class GetSequencesEndpointTest(
         val sequencesOfUser = convenienceClient.getSequenceEntries(
             username = DEFAULT_USER_NAME,
             organism = OTHER_ORGANISM,
-        )
+        ).sequenceEntries
 
         assertThat(
             sequencesOfUser.getAccessionVersions(),
@@ -124,24 +128,73 @@ class GetSequencesEndpointTest(
         val sequencesInAwaitingApproval = convenienceClient.getSequenceEntries(
             username = ALTERNATIVE_DEFAULT_USER_NAME,
             statusesFilter = listOf(AWAITING_APPROVAL),
-        )
+        ).sequenceEntries
 
         assertThat(sequencesInAwaitingApproval, hasSize(10))
 
         val sequencesInProcessing = convenienceClient.getSequenceEntries(
             username = ALTERNATIVE_DEFAULT_USER_NAME,
             statusesFilter = listOf(IN_PROCESSING),
-        )
+        ).sequenceEntries
 
         assertThat(sequencesInProcessing, hasSize(0))
     }
 
+    @Test
+    fun `GIVEN data in many statuses WHEN querying sequences with pagination THEN return paged results`() {
+        val allSubmittedSequencesSorted = convenienceClient.prepareDataTo(AWAITING_APPROVAL).map {
+            it.accession to it.version
+        }.sortedBy { it.first }
+        convenienceClient.prepareDataTo(HAS_ERRORS)
+
+        val resultForInAwaitingApprovalPageOne = convenienceClient.getSequenceEntries(
+            statusesFilter = listOf(AWAITING_APPROVAL),
+            page = 0,
+            size = 5,
+        )
+
+        assertThat(resultForInAwaitingApprovalPageOne.sequenceEntries, hasSize(5))
+        assertThat(resultForInAwaitingApprovalPageOne.statusCounts, `is`(mapOf(AWAITING_APPROVAL to 10)))
+
+        val resultForInAwaitingApprovalPageTwo = convenienceClient.getSequenceEntries(
+            statusesFilter = listOf(AWAITING_APPROVAL),
+            page = 1,
+            size = 5,
+        )
+
+        val combinedResultSorted = (
+            resultForInAwaitingApprovalPageOne.sequenceEntries +
+                resultForInAwaitingApprovalPageTwo.sequenceEntries
+            ).map { it.accession to it.version }.sortedBy { it.first }
+
+        assertThat(combinedResultSorted, `is`(allSubmittedSequencesSorted))
+
+        val generalResult = convenienceClient.getSequenceEntries()
+
+        assertThat(generalResult.sequenceEntries, hasSize(20))
+        assertThat(
+            generalResult.statusCounts,
+            `is`(
+                mapOf(
+                    RECEIVED to 0,
+                    IN_PROCESSING to 0,
+                    AWAITING_APPROVAL to 10,
+                    HAS_ERRORS to 10,
+                    APPROVED_FOR_RELEASE to 0,
+                    AWAITING_APPROVAL_FOR_REVOCATION to 0,
+                ),
+            ),
+        )
+    }
+
     @ParameterizedTest(name = "{arguments}")
     @MethodSource("provideStatusScenarios")
     fun `GIVEN database in prepared state THEN returns sequence entries in expected status`(scenario: Scenario) {
         scenario.prepareDatabase(convenienceClient)
 
-        val sequencesOfUser = convenienceClient.getSequenceEntries(statusesFilter = listOf(scenario.expectedStatus))
+        val sequencesOfUser = convenienceClient.getSequenceEntries(
+            statusesFilter = listOf(scenario.expectedStatus),
+        ).sequenceEntries
 
         val accessionVersionStatus =
             sequencesOfUser.find { it.accession == firstAccession && it.version == scenario.expectedVersion }
@@ -155,7 +208,7 @@ class GetSequencesEndpointTest(
             Scenario(
                 setupDescription = "I submitted sequence entries",
                 prepareDatabase = { it.submitDefaultFiles() },
-                expectedStatus = Status.RECEIVED,
+                expectedStatus = RECEIVED,
                 expectedIsRevocation = false,
             ),
             Scenario(
@@ -186,7 +239,7 @@ class GetSequencesEndpointTest(
                     it.prepareDatabaseWithProcessedData(PreparedProcessedData.successfullyProcessed())
                     it.approveProcessedSequenceEntries(listOf(AccessionVersion(firstAccession, 1)))
                 },
-                expectedStatus = Status.APPROVED_FOR_RELEASE,
+                expectedStatus = APPROVED_FOR_RELEASE,
                 expectedIsRevocation = false,
             ),
             Scenario(
@@ -196,7 +249,7 @@ class GetSequencesEndpointTest(
                     it.approveProcessedSequenceEntries(listOf(AccessionVersion(firstAccession, 1)))
                     it.revokeSequenceEntries(listOf(firstAccession))
                 },
-                expectedStatus = Status.AWAITING_APPROVAL_FOR_REVOCATION,
+                expectedStatus = AWAITING_APPROVAL_FOR_REVOCATION,
                 expectedIsRevocation = true,
                 expectedVersion = 2,
             ),
@@ -208,7 +261,7 @@ class GetSequencesEndpointTest(
                     it.revokeSequenceEntries(listOf(firstAccession))
                     it.confirmRevocation(listOf(AccessionVersion(firstAccession, 2)))
                 },
-                expectedStatus = Status.APPROVED_FOR_RELEASE,
+                expectedStatus = APPROVED_FOR_RELEASE,
                 expectedIsRevocation = true,
                 expectedVersion = 2,
             ),

backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt

@@ -84,12 +84,16 @@ class SubmissionControllerClient(private val mockMvc: MockMvc, private val objec
         groupsFilter: List<String>? = null,
         statusesFilter: List<Status>? = null,
         jwt: String? = jwtForDefaultUser,
+        page: Int? = null,
+        size: Int? = null,
     ): ResultActions {
         return mockMvc.perform(
             get(addOrganismToPath("/get-sequences", organism = organism))
                 .withAuth(jwt)
                 .param("groupsFilter", groupsFilter?.joinToString { it })
-                .param("statusesFilter", statusesFilter?.joinToString { it.name }),
+                .param("statusesFilter", statusesFilter?.joinToString { it.name })
+                .param("page", page?.toString())
+                .param("size", size?.toString()),
         )
     }