feat(backend): handle timestamp metadata field #755

Make them an integer in SILO - display them as a human-readable date in the frontend.

backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt

@@ -2,6 +2,9 @@ package org.loculus.backend.model
 
 import com.fasterxml.jackson.databind.node.LongNode
 import com.fasterxml.jackson.databind.node.TextNode
+import kotlinx.datetime.LocalDateTime
+import kotlinx.datetime.TimeZone
+import kotlinx.datetime.toInstant
 import mu.KotlinLogging
 import org.loculus.backend.api.Organism
 import org.loculus.backend.api.ProcessedData
@@ -42,8 +45,8 @@ class ReleasedDataModel(private val submissionDatabaseService: SubmissionDatabas
             ("accessionVersion" to TextNode(rawProcessedData.displayAccessionVersion())) +
             ("isRevocation" to TextNode(rawProcessedData.isRevocation.toString())) +
             ("submitter" to TextNode(rawProcessedData.submitter)) +
-            ("submittedAt" to TextNode(rawProcessedData.submittedAt.toString())) +
-            ("releasedAt" to TextNode(rawProcessedData.releasedAt.toString())) +
+            ("submittedAt" to LongNode(rawProcessedData.submittedAt.toTimestamp())) +
+            ("releasedAt" to LongNode(rawProcessedData.releasedAt.toTimestamp())) +
             ("versionStatus" to TextNode(siloVersionStatus.name))
 
         return ProcessedData(
@@ -75,3 +78,5 @@ class ReleasedDataModel(private val submissionDatabaseService: SubmissionDatabas
         return SiloVersionStatus.REVISED
     }
 }
+
+private fun LocalDateTime.toTimestamp() = this.toInstant(TimeZone.UTC).epochSeconds

backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt

@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.node.IntNode
 import com.fasterxml.jackson.databind.node.NullNode
 import com.fasterxml.jackson.databind.node.TextNode
 import kotlinx.datetime.Clock
+import kotlinx.datetime.Instant
 import kotlinx.datetime.TimeZone
 import kotlinx.datetime.toLocalDateTime
 import org.hamcrest.CoreMatchers.`is`
@@ -24,8 +25,6 @@ import org.loculus.backend.controller.submission.SubmitFiles.DefaultFiles.firstA
 import org.loculus.backend.utils.Accession
 import org.loculus.backend.utils.Version
 import org.springframework.beans.factory.annotation.Autowired
-import java.time.LocalDateTime
-import java.time.format.DateTimeFormatter
 
 private val ADDED_FIELDS_WITH_UNKNOWN_VALUES_FOR_RELEASE = listOf("releasedAt", "submissionId", "submittedAt")
 
@@ -91,8 +90,8 @@ class GetReleasedDataEndpointTest(
             )
             for ((key, value) in it.metadata) {
                 when (key) {
-                    "submittedAt" -> expectIsDateWithCurrentYear(value)
-                    "releasedAt" -> expectIsDateWithCurrentYear(value)
+                    "submittedAt" -> expectIsTimestampWithCurrentYear(value)
+                    "releasedAt" -> expectIsTimestampWithCurrentYear(value)
                     "submissionId" -> assertThat(value.textValue(), matchesPattern("^custom\\d$"))
                     else -> assertThat(value, `is`(expectedMetadata[key]))
                 }
@@ -176,8 +175,8 @@ class GetReleasedDataEndpointTest(
             when (key) {
                 "isRevocation" -> assertThat(value, `is`(TextNode("true")))
                 "versionStatus" -> assertThat(value, `is`(TextNode("LATEST_VERSION")))
-                "submittedAt" -> expectIsDateWithCurrentYear(value)
-                "releasedAt" -> expectIsDateWithCurrentYear(value)
+                "submittedAt" -> expectIsTimestampWithCurrentYear(value)
+                "releasedAt" -> expectIsTimestampWithCurrentYear(value)
                 "submitter" -> assertThat(value, `is`(TextNode(DEFAULT_USER_NAME)))
                 "accession", "version", "accessionVersion", "submissionId" -> {}
                 else -> assertThat("value for $key", value, `is`(NullNode.instance))
@@ -229,8 +228,8 @@ class GetReleasedDataEndpointTest(
         )
     }
 
-    private fun expectIsDateWithCurrentYear(value: JsonNode) {
-        val dateTime = LocalDateTime.parse(value.textValue(), DateTimeFormatter.ISO_LOCAL_DATE_TIME)
+    private fun expectIsTimestampWithCurrentYear(value: JsonNode) {
+        val dateTime = Instant.fromEpochSeconds(value.asLong()).toLocalDateTime(TimeZone.UTC)
         assertThat(dateTime.year, `is`(currentYear))
     }
 }

kubernetes/loculus/templates/_common-metadata.tpl

@@ -16,9 +16,9 @@ fields:
   - name: submitter
     type: string
   - name: submittedAt
-    type: string
+    type: timestamp
   - name: releasedAt
-    type: string
+    type: timestamp
   - name: versionStatus
     type: string
     notSearchable: true

kubernetes/loculus/templates/lapis-silo-database-config.yaml

@@ -17,7 +17,7 @@ data:
       metadata:
       {{- range (concat $commonMetadata .metadata) }}
         - name: {{ .name }}
-          type: {{ .type }}
+          type: {{ (.type | eq "timestamp") | ternary "int" .type }}
           {{- if .generateIndex }}
           generateIndex: {{ .generateIndex }}
           {{- end }}

website/src/components/SearchPage/SearchForm.spec.tsx

@@ -92,4 +92,42 @@ describe('SearchForm', () => {
         expect(screen.getByPlaceholderText('Field 1')).toBeDefined();
         expect(screen.queryByPlaceholderText('NotSearchable')).not.toBeInTheDocument();
     });
+
+    test('should display dates of timestamp fields', async () => {
+        const timestampFieldName = 'timestampField';
+        renderSearchForm([
+            {
+                name: timestampFieldName,
+                type: 'timestamp' as const,
+                filterValue: '1706147200',
+            },
+        ]);
+
+        const timestampField = screen.getByLabelText('Timestamp field');
+        expect(timestampField).toHaveValue('2024-01-25');
+
+        await userEvent.type(timestampField, '2024-01-26');
+        await userEvent.click(screen.getByRole('button', { name: 'Search' }));
+
+        expect(window.location.href).toContain(`${timestampFieldName}=1706233600`);
+    });
+
+    test('should display dates of date fields', async () => {
+        const dateFieldName = 'dateField';
+        renderSearchForm([
+            {
+                name: dateFieldName,
+                type: 'date' as const,
+                filterValue: '2024-01-25',
+            },
+        ]);
+
+        const dateField = screen.getByLabelText('Date field');
+        expect(dateField).toHaveValue('2024-01-25');
+
+        await userEvent.type(dateField, '2024-01-26');
+        await userEvent.click(screen.getByRole('button', { name: 'Search' }));
+
+        expect(window.location.href).toContain(`${dateFieldName}=2024-01-26`);
+    });
 });

website/src/components/SearchPage/SearchForm.tsx

@@ -6,7 +6,8 @@ import { sentenceCase } from 'change-case';
 import { type FC, type FormEventHandler, useMemo, useState } from 'react';
 
 import { AutoCompleteField } from './fields/AutoCompleteField';
-import { DateField } from './fields/DateField';
+import { DateField, TimestampField } from './fields/DateField';
+import type { FieldProps } from './fields/FieldProps.tsx';
 import { MutationField } from './fields/MutationField.tsx';
 import { NormalTextField } from './fields/NormalTextField';
 import { PangoLineageField } from './fields/PangoLineageField';
@@ -77,30 +78,16 @@ export const SearchForm: FC<SearchFormProps> = ({
 
     const fields = useMemo(
         () =>
-            fieldValues.map((field) => {
-                if (field.notSearchable === true) return null;
-
-                const props = {
-                    key: field.name,
-                    field,
-                    handleFieldChange,
-                    isLoading,
-                    lapisUrl,
-                    allFields: fieldValues,
-                };
-
-                switch (field.type) {
-                    case 'date':
-                        return <DateField {...props} />;
-                    case 'pango_lineage':
-                        return <PangoLineageField {...props} />;
-                    default:
-                        if (field.autocomplete === true) {
-                            return <AutoCompleteField {...props} />;
-                        }
-                        return <NormalTextField {...props} />;
-                }
-            }),
+            fieldValues.map((field) => (
+                <SearchField
+                    key={field.name}
+                    field={field}
+                    handleFieldChange={handleFieldChange}
+                    isLoading={isLoading}
+                    lapisUrl={lapisUrl}
+                    allFields={fieldValues}
+                />
+            )),
         [lapisUrl, fieldValues, isLoading],
     );
 
@@ -155,6 +142,28 @@ export const SearchForm: FC<SearchFormProps> = ({
     );
 };
 
+const SearchField: FC<FieldProps> = (props) => {
+    const { field } = props;
+
+    if (field.notSearchable === true) {
+        return null;
+    }
+
+    switch (field.type) {
+        case 'date':
+            return <DateField {...props} />;
+        case 'timestamp':
+            return <TimestampField {...props} />;
+        case 'pango_lineage':
+            return <PangoLineageField {...props} />;
+        default:
+            if (field.autocomplete === true) {
+                return <AutoCompleteField {...props} />;
+            }
+            return <NormalTextField {...props} />;
+    }
+};
+
 const SearchButton: FC<{ isLoading: boolean }> = ({ isLoading }) => (
     <button className='btn normal-case w-full' type='submit' disabled={isLoading}>
         {isLoading ? <CircularProgress size={20} color='primary' /> : 'Search'}

website/src/components/SearchPage/fields/DateField.tsx

@@ -4,7 +4,37 @@ import type { FC } from 'react';
 
 import type { FieldProps } from './FieldProps';
 
-export const DateField: FC<FieldProps> = ({ field, handleFieldChange, isLoading }) => (
+type ValueConverter = {
+    dateToValueConverter: (date: DateTime | null) => string;
+    valueToDateConverter: (value: string) => DateTime | null;
+};
+
+export const DateField: FC<FieldProps> = (props) => (
+    <CustomizedDatePicker
+        {...props}
+        dateToValueConverter={(date) => date?.toISODate() ?? ''}
+        valueToDateConverter={(value) => (value === '' ? null : DateTime.fromISO(value))}
+    />
+);
+
+export const TimestampField: FC<FieldProps> = (props) => (
+    <CustomizedDatePicker
+        {...props}
+        dateToValueConverter={(date) => date?.toSeconds().toString() ?? ''}
+        valueToDateConverter={(value) => {
+            const timestamp = Number(value);
+            return timestamp > 0 ? DateTime.fromSeconds(timestamp) : null;
+        }}
+    />
+);
+
+const CustomizedDatePicker: FC<FieldProps & ValueConverter> = ({
+    field,
+    handleFieldChange,
+    isLoading,
+    dateToValueConverter,
+    valueToDateConverter,
+}) => (
     <DatePicker
         format='yyyy-MM-dd'
         label={field.label}
@@ -15,10 +45,9 @@ export const DateField: FC<FieldProps> = ({ field, handleFieldChange, isLoading
                 margin: 'dense',
             },
         }}
-        value={field.filterValue === '' ? null : DateTime.fromISO(field.filterValue)}
+        value={valueToDateConverter(field.filterValue)}
         onChange={(date: DateTime | null) => {
-            const dateString = date?.toISODate() ?? '';
-            return handleFieldChange(field.name, dateString);
+            return handleFieldChange(field.name, dateToValueConverter(date));
         }}
     />
 );

website/src/components/SequenceDetailsPage/getTableData.spec.ts

@@ -13,6 +13,7 @@ const schema: Schema = {
     metadata: [
         { name: 'metadataField1', type: 'string' },
         { name: 'metadataField2', type: 'string' },
+        { name: 'timestampField', type: 'timestamp' },
     ],
     tableColumns: [],
     primaryKey: 'primary key',
@@ -80,6 +81,11 @@ describe('getTableData', () => {
                     name: 'metadataField2',
                     value: 'N/A',
                 },
+                {
+                    label: 'Timestamp field',
+                    name: 'timestampField',
+                    value: 'N/A',
+                },
                 {
                     label: 'Nucleotide substitutions',
                     name: 'nucleotideSubstitutions',
@@ -189,6 +195,19 @@ describe('getTableData', () => {
             value: 'aminoAcidInsertion1, aminoAcidInsertion2',
         });
     });
+
+    test('should map timestamps to human readable dates', async () => {
+        mockRequest.lapis.details(200, { data: [{ timestampField: 1706194761 }] });
+
+        const result = await getTableData('accession', schema, lapisClient);
+
+        const data = result._unsafeUnwrap();
+        expect(data).toContainEqual({
+            label: 'Timestamp field',
+            name: 'timestampField',
+            value: '2024-01-25 14:59:21 UTC',
+        });
+    });
 });
 
 describe('getVersionStatus', () => {

website/src/components/SequenceDetailsPage/getTableData.ts

@@ -1,10 +1,11 @@
 import { sentenceCase } from 'change-case';
+import { DateTime, FixedOffsetZone } from 'luxon';
 import { err, Result } from 'neverthrow';
 
 import { type LapisClient } from '../../services/lapisClient.ts';
 import { VERSION_STATUS_FIELD } from '../../settings.ts';
 import type { AccessionVersion, ProblemDetail } from '../../types/backend.ts';
-import type { Schema } from '../../types/config.ts';
+import type { Metadata, Schema } from '../../types/config.ts';
 import {
     type Details,
     type DetailsResponse,
@@ -106,8 +107,9 @@ function toTableData(config: Schema) {
         const data: TableDataEntry[] = config.metadata.map((metadata) => ({
             label: sentenceCase(metadata.name),
             name: metadata.name,
-            value: details[metadata.name] ?? 'N/A',
+            value: mapValueToDisplayedValue(details[metadata.name], metadata),
         }));
+
         data.push(
             {
                 label: 'Nucleotide substitutions',
@@ -145,6 +147,18 @@ function toTableData(config: Schema) {
     };
 }
 
+function mapValueToDisplayedValue(value: undefined | null | string | number, metadata: Metadata) {
+    if (value === null || value === undefined) {
+        return 'N/A';
+    }
+
+    if (metadata.type === 'timestamp' && typeof value === 'number') {
+        return DateTime.fromSeconds(value, { zone: FixedOffsetZone.utcInstance }).toFormat('yyyy-MM-dd TTT');
+    }
+
+    return value;
+}
+
 function mutationsToCommaSeparatedString(
     mutationData: MutationProportionCount[],
     filter: (mutation: string) => boolean,

website/src/pages/[organism]/search/search.ts

@@ -5,7 +5,7 @@ import { getReferenceGenomes, getSchema } from '../../../config.ts';
 import { LapisClient } from '../../../services/lapisClient.ts';
 import { hiddenDefaultSearchFilters } from '../../../settings.ts';
 import type { ProblemDetail } from '../../../types/backend.ts';
-import type { MetadataFilter, FilterValue, MutationFilter } from '../../../types/config.ts';
+import type { FilterValue, MetadataFilter, MutationFilter } from '../../../types/config.ts';
 import { type LapisBaseRequest, type OrderBy, type OrderByType, orderByType } from '../../../types/lapis.ts';
 import type { ReferenceGenomesSequenceNames } from '../../../types/referencesGenomes.ts';
 
@@ -83,7 +83,8 @@ export const getMetadataFilters = (getSearchParams: (param: string) => string, o
         if (metadata.notSearchable === true) {
             return [];
         }
-        if (metadata.type === 'date') {
+
+        if (metadata.type === 'date' || metadata.type === 'timestamp') {
             const metadataFrom = {
                 ...metadata,
                 name: `${metadata.name}From`,
@@ -95,13 +96,14 @@ export const getMetadataFilters = (getSearchParams: (param: string) => string, o
                 filterValue: getSearchParams(`${metadata.name}To`),
             };
             return [metadataFrom, metadataTo];
-        } else {
-            const metadataSetting = {
+        }
+
+        return [
+            {
                 ...metadata,
                 filterValue: getSearchParams(metadata.name),
-            };
-            return [metadataSetting];
-        }
+            },
+        ];
     });
 };