Allow very long search queries (#967)

* wip * enablePreprocessing * wip * update * typing etc. fixes * type fixes * updates * u * fix * add test for POST function * remove test * Update website/src/routes.ts Co-authored-by: Fabian Engelniederhammer <92720311+fengelniederhammer@users.noreply.github.com> * simplify * Update DataUploadForm.tsx * Update setup_codespace_env.sh * Update DataUploadForm.tsx * Automated code formatting --------- Co-authored-by: Fabian Engelniederhammer <92720311+fengelniederhammer@users.noreply.github.com> Co-authored-by: Loculus bot <bot@loculus.org>
loculus-project · Feb 12, 2024 · 7408255 · 7408255
1 parent cc80095
commit 7408255
Show file tree

Hide file tree

Showing 7 changed files with 125 additions and 51 deletions.
diff --git a/website/src/components/DataUploadForm.tsx b/website/src/components/DataUploadForm.tsx
@@ -45,6 +45,7 @@ const InnerDataUploadForm = ({
 }: DataUploadFormProps) => {
     const [metadataFile, setMetadataFile] = useState<File | null>(null);
     const [sequenceFile, setSequenceFile] = useState<File | null>(null);
+    const [exampleEntries, setExampleEntries] = useState<number | undefined>(undefined);
     const metadataFileInputRef = useRef<HTMLInputElement>(null);
     const sequenceFileInputRef = useRef<HTMLInputElement>(null);
 
@@ -68,7 +69,7 @@ const InnerDataUploadForm = ({
     const [restrictedUntil, setRestrictedUntil] = useState<DateTime>(dateTimeInMonths(6));
 
     const handleLoadExampleData = async () => {
-        const { metadataFileContent, revisedMetadataFileContent, sequenceFileContent } = getExampleData();
+        const { metadataFileContent, revisedMetadataFileContent, sequenceFileContent } = getExampleData(exampleEntries);
 
         const exampleMetadataContent = action === `submit` ? metadataFileContent : revisedMetadataFileContent;
 
@@ -248,9 +249,18 @@ const InnerDataUploadForm = ({
 
             <div className='flex gap-4'>
                 {organism.startsWith('dummy-organism') && (
-                    <button type='button' className='px-4 py-2 btn normal-case ' onClick={handleLoadExampleData}>
-                        Load Example Data
-                    </button>
+                    <>
+                        <input
+                            type='number'
+                            className='p-1 border rounded-md w-28 placeholder:text-xs'
+                            placeholder='num of examples'
+                            value={exampleEntries ?? ''}
+                            onChange={(event) => setExampleEntries(parseInt(event.target.value, 10))}
+                        />
+                        <button type='button' className='px-4 py-2 btn normal-case ' onClick={handleLoadExampleData}>
+                            Load Example Data
+                        </button>
+                    </>
                 )}
 
                 <button
@@ -317,29 +327,35 @@ function handleError(onError: (message: string) => void, action: Action) {
     };
 }
 
-function getExampleData() {
+function getExampleData(randomEntries = 20) {
+    const regions = ['Europe', 'Asia', 'North America', 'South America', 'Africa', 'Australia'];
+    const countries = ['Switzerland', 'USA', 'China', 'Brazil', 'Nigeria', 'Australia'];
+    const divisions = ['Bern', 'California', 'Beijing', 'Rio de Janeiro', 'Lagos', 'Sydney'];
+    const hosts = ['Homo sapiens', 'Canis lupus familiaris'];
+
+    let metadataContent = 'submissionId\tdate\tregion\tcountry\tdivision\thost\n';
+    let revisedMetadataContent = 'accession\tsubmissionId\tdate\tregion\tcountry\tdivision\thost\n';
+    let sequenceContent = '';
+
+    for (let i = 0; i < randomEntries; i++) {
+        const submissionId = `custom${i}`;
+        const date = new Date(Date.now() - Math.floor(Math.random() * 365 * 24 * 60 * 60 * 1000))
+            .toISOString()
+            .split('T')[0];
+        const region = regions[Math.floor(Math.random() * regions.length)];
+        const country = countries[Math.floor(Math.random() * countries.length)];
+        const division = divisions[Math.floor(Math.random() * divisions.length)];
+        const host = hosts[Math.floor(Math.random() * hosts.length)];
+
+        metadataContent += `${submissionId}\t${date}\t${region}\t${country}\t${division}\t${host}\n`;
+        revisedMetadataContent += `${i + 1}\t${submissionId}\t${date}\t${region}\t${country}\t${division}\t${host}\n`;
+        sequenceContent += `>${submissionId}\nACTG\n`;
+    }
+
     return {
-        metadataFileContent: `
-submissionId	date	region	country	division	host
-custom0	2020-12-26	Europe	Switzerland	Bern	Homo sapiens
-custom1	2020-12-15	Europe	Switzerland	Schaffhausen	Homo sapiens
-custom2	2020-12-02	Europe	Switzerland	Bern	Homo sapiens
-custom3	2020-12-02	Europe	Switzerland	Bern	Homo sapiens`,
-        revisedMetadataFileContent: `
-accession	submissionId	date	region	country	division	host
-1	custom0	2020-12-26	Europe	Switzerland	Bern	Homo sapiens
-2	custom1	2020-12-15	Europe	Switzerland	Schaffhausen	Homo sapiens
-3	custom2	2020-12-02	Europe	Switzerland	Bern	Homo sapiens
-4	custom3	2020-12-02	Europe	Switzerland	Bern	Homo sapiens`,
-        sequenceFileContent: `
->custom0
-ACTG
->custom1
-ACTG
->custom2
-ACTG
->custom3
-ACTG`,
+        metadataFileContent: metadataContent,
+        revisedMetadataFileContent: revisedMetadataContent,
+        sequenceFileContent: sequenceContent,
     };
 }
 

diff --git a/website/src/components/SearchPage/Pagination.tsx b/website/src/components/SearchPage/Pagination.tsx
@@ -1,22 +1,26 @@
 import { Pagination as MUIPagination } from '@mui/material';
 import type { FC } from 'react';
 
+import { navigateToSearchPage } from '../../routes';
+import type { MetadataFilter, MutationFilter } from '../../types/config.ts';
+import type { OrderBy } from '../../types/lapis.ts';
+
 type Props = {
     count: number;
+    metadataFilter: MetadataFilter[];
+    mutationFilter: MutationFilter;
+    orderBy: OrderBy;
+    organism: string;
+    page: number;
 };
 
-export const Pagination: FC<Props> = ({ count }) => {
-    const params = new URLSearchParams(location.search);
-    const pageParam = params.get('page');
-    const page = pageParam !== null ? Number.parseInt(pageParam, 10) : 1;
-
+export const Pagination: FC<Props> = ({ count, metadataFilter, mutationFilter, orderBy, organism, page }) => {
     return (
         <MUIPagination
             count={count}
             page={page}
             onChange={(_, newPage) => {
-                params.set('page', newPage.toString());
-                location.href = `search?${params}`;
+                navigateToSearchPage(organism, metadataFilter, mutationFilter, newPage, orderBy);
             }}
             color='primary'
             variant='outlined'

diff --git a/website/src/components/SearchPage/SearchForm.tsx b/website/src/components/SearchPage/SearchForm.tsx
@@ -14,7 +14,7 @@ import { PangoLineageField } from './fields/PangoLineageField';
 import { getClientLogger } from '../../clientLogger.ts';
 import { getLapisUrl } from '../../config.ts';
 import { useOffCanvas } from '../../hooks/useOffCanvas';
-import { routes } from '../../routes.ts';
+import { routes, navigateToSearchPage } from '../../routes.ts';
 import type { MetadataFilter, MutationFilter } from '../../types/config.ts';
 import type { ReferenceGenomesSequenceNames } from '../../types/referencesGenomes.ts';
 import type { ClientConfig } from '../../types/runtimeConfig.ts';
@@ -66,7 +66,7 @@ export const SearchForm: FC<SearchFormProps> = ({
         event.preventDefault();
         setIsLoading(true);
         const searchableFieldValues = fieldValues.filter((field) => !(field.notSearchable ?? false));
-        location.href = routes.searchPage(organism, searchableFieldValues, mutationFilter);
+        navigateToSearchPage(organism, searchableFieldValues, mutationFilter);
     };
 
     const resetSearch = async () => {

diff --git a/website/src/components/SearchPage/Table.tsx b/website/src/components/SearchPage/Table.tsx
@@ -1,7 +1,7 @@
 import { capitalCase } from 'change-case';
 import type { FC, ReactElement } from 'react';
 
-import { routes } from '../../routes.ts';
+import { routes, navigateToSearchPage } from '../../routes.ts';
 import type { MetadataFilter, MutationFilter, Schema } from '../../types/config.ts';
 import type { OrderBy } from '../../types/lapis.ts';
 import MdiTriangle from '~icons/mdi/triangle';
@@ -32,18 +32,18 @@ export const Table: FC<TableProps> = ({ organism, data, schema, metadataFilter,
     const handleSort = (field: string) => {
         if (orderBy.field === field) {
             if (orderBy.type === 'ascending') {
-                location.href = routes.searchPage(organism, metadataFilter, mutationFilter, page, {
+                navigateToSearchPage(organism, metadataFilter, mutationFilter, page, {
                     field,
                     type: 'descending',
                 });
             } else {
-                location.href = routes.searchPage(organism, metadataFilter, mutationFilter, page, {
+                navigateToSearchPage(organism, metadataFilter, mutationFilter, page, {
                     field,
                     type: 'ascending',
                 });
             }
         } else {
-            location.href = routes.searchPage(organism, metadataFilter, mutationFilter, page, {
+            navigateToSearchPage(organism, metadataFilter, mutationFilter, page, {
                 field,
                 type: 'ascending',
             });

diff --git a/website/src/pages/[organism]/search/index.astro b/website/src/pages/[organism]/search/index.astro
@@ -22,17 +22,29 @@ const { organism: cleanedOrganism } = cleanOrganism(organism);
 const schema = getSchema(organism);
 const clientConfig = getRuntimeConfig().public;
 const lapisUrl = getLapisUrl(clientConfig, organism);
+let postParams = new URLSearchParams();
+
+if (Astro.request.method === 'POST') {
+    const decoded = await Astro.request.formData();
+    const paramsString = decoded.searchQuery;
+    postParams = new URLSearchParams(paramsString);
+}
+
 const getSearchParams = (field: string): string => {
-    return Astro.url.searchParams.get(field) ?? '';
+    let value = Astro.url.searchParams.get(field);
+    if (value === null && postParams.get(field) !== null) {
+        value = postParams.get(field);
+    }
+    return value ?? '';
 };
 
 const metadataFilter = addHiddenFilters(getMetadataFilters(getSearchParams, organism), hiddenDefaultSearchFilters);
 const mutationFilter = getMutationFilter(Astro.url.searchParams);
 
-const pageParam = Astro.url.searchParams.get('page');
-const page = pageParam !== null ? Number.parseInt(pageParam, 10) : 1;
+const pageParam = getSearchParams('page');
+const page = pageParam !== '' ? Number.parseInt(pageParam, 10) : 1;
 const offset = (page - 1) * pageSize;
-const orderBy = getOrderBy(Astro.url.searchParams, schema.defaultOrderBy, schema.defaultOrder);
+const orderBy = getOrderBy(getSearchParams, schema.defaultOrderBy, schema.defaultOrder);
 
 const referenceGenomesSequenceNames = getReferenceGenomesSequenceNames(organism);
 
@@ -79,7 +91,15 @@ const data = await getData(organism, metadataFilter, mutationFilter, offset, pag
                         />
 
                         <div class='mt-4 flex justify-center'>
-                            <Pagination client:only='react' count={Math.ceil(data.totalCount / pageSize)} />
+                            <Pagination
+                                client:only='react'
+                                count={Math.ceil(data.totalCount / pageSize)}
+                                page={page}
+                                metadataFilter={metadataFilter}
+                                mutationFilter={mutationFilter}
+                                orderBy={orderBy}
+                                organism={organism}
+                            />
                         </div>
                     </div>
                 ),

diff --git a/website/src/pages/[organism]/search/search.ts b/website/src/pages/[organism]/search/search.ts
@@ -104,14 +104,14 @@ export const getMetadataFilters = (getSearchParams: (param: string) => string, o
 };
 
 export const getOrderBy = (
-    searchParams: URLSearchParams,
+    getSearchParams: (param: string) => string,
     defaultOrderByField: string,
     defaultOrder: OrderByType,
 ): OrderBy => {
-    const orderByTypeParam = searchParams.get('order');
-    const orderByTypeParsed = orderByTypeParam !== null ? orderByType.safeParse(orderByTypeParam) : undefined;
+    const orderByTypeParam = getSearchParams('order');
+    const orderByTypeParsed = orderByTypeParam !== '' ? orderByType.safeParse(orderByTypeParam) : undefined;
     const orderByTypeValue: OrderByType = orderByTypeParsed?.success === true ? orderByTypeParsed.data : defaultOrder;
-    const sortByField = searchParams.get('orderBy') ?? defaultOrderByField;
+    const sortByField = getSearchParams('orderBy') ? getSearchParams('orderBy') : defaultOrderByField;
     return {
         field: sortByField,
         type: orderByTypeValue,

diff --git a/website/src/routes.ts b/website/src/routes.ts
@@ -3,6 +3,8 @@ import type { FilterValue, MutationFilter } from './types/config.ts';
 import type { OrderBy } from './types/lapis.ts';
 import { getAccessionVersionString } from './utils/extractAccessionVersion.ts';
 
+const approxMaxUrlLengthForSearch = 1900;
+
 export const routes = {
     aboutPage: () => '/about',
     apiDocumentationPage: () => '/api_documentation',
@@ -13,7 +15,7 @@ export const routes = {
         organism: string,
         metadataFilter: Filter[] = [],
         mutationFilter: MutationFilter = {},
-        page: number = 1,
+        page: number | undefined = undefined,
         orderBy?: OrderBy,
     ) =>
         withOrganism(
@@ -54,11 +56,41 @@ export const routes = {
     notFoundPage: () => `/404`,
     logout: () => '/logout',
 };
+export const navigateToSearchPage = (
+    organism: string,
+    metadataFilter: FilterValue[] = [],
+    mutationFilter: MutationFilter = {},
+    page?: number,
+    orderBy?: OrderBy,
+) => {
+    const paramsString = buildSearchParams(metadataFilter, mutationFilter, page, orderBy).toString();
+
+    if (paramsString.length < approxMaxUrlLengthForSearch) {
+        location.href = routes.searchPage(organism, metadataFilter, mutationFilter, page, orderBy);
+    } else {
+        const form = document.createElement('form');
+        const addField = (name: string, value: string) => {
+            const field = document.createElement('input');
+            field.type = 'hidden';
+            field.name = name;
+            field.value = value;
+            form.appendChild(field);
+        };
+        form.method = 'POST';
+        form.action = routes.searchPage(organism);
+
+        addField('searchQuery', paramsString);
+        addField('organism', organism);
+
+        document.body.appendChild(form);
+        form.submit();
+    }
+};
 
 const buildSearchParams = <Filter extends FilterValue>(
     metadataFilter: Filter[],
     mutationFilter: MutationFilter,
-    page: number,
+    page?: number,
     orderBy?: OrderBy,
 ) => {
     const params = new URLSearchParams();
@@ -86,7 +118,9 @@ const buildSearchParams = <Filter extends FilterValue>(
         params.set('orderBy', orderBy.field);
         params.set('order', orderBy.type);
     }
-    params.set('page', page.toString());
+    if (page !== undefined) {
+        params.set('page', page.toString());
+    }
     return params;
 };