From 1b9514f003c2f477572eeab558aec981654a1406 Mon Sep 17 00:00:00 2001 From: Steve Cassidy Date: Mon, 30 Sep 2024 12:28:09 +1000 Subject: [PATCH 1/6] fix lat/long order in csv export and add accuracy, created, created_by Signed-off-by: Steve Cassidy --- api/src/couchdb/notebooks.ts | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/api/src/couchdb/notebooks.ts b/api/src/couchdb/notebooks.ts index 058635572..0e57db838 100644 --- a/api/src/couchdb/notebooks.ts +++ b/api/src/couchdb/notebooks.ts @@ -623,16 +623,35 @@ const csvFormatValue = ( if (fieldType === 'faims-pos::Location') { if (value instanceof Object && 'geometry' in value) { result[fieldName] = value; - result[fieldName + '_latitude'] = value.geometry.coordinates[0]; - result[fieldName + '_longitude'] = value.geometry.coordinates[1]; + result[fieldName + '_latitude'] = value.geometry.coordinates[1]; + result[fieldName + '_longitude'] = value.geometry.coordinates[0]; + result[fieldName + '_accuracy'] = value.properties.accuracy || ''; } else { result[fieldName] = value; result[fieldName + '_latitude'] = ''; result[fieldName + '_longitude'] = ''; + result[fieldName + '_accuracy'] = ''; } return result; } + if (fieldType === 'faims-core::JSON') { + // map location, if it's a point we can pull out lat/long + if ( + value instanceof Object && + 'features' in value && + value.features.length > 0 && + value.features[0]?.geometry?.type === 'Point' + ) { + result[fieldName] = value; + result[fieldName + '_latitude'] = + value.features[0].geometry.coordinates[1]; + result[fieldName + '_longitude'] = + value.features[0].geometry.coordinates[0]; + return result; + } + } + if (fieldType === 'faims-core::Relationship') { if (value instanceof Array) { result[fieldName] = value @@ -740,6 +759,8 @@ export const streamNotebookRecordsAsCSV = async ( record.record_id, record.revision_id, record.type, + record.created_by, + record.created.toISOString(), record.updated_by, record.updated.toISOString(), ]; @@ -759,6 +780,8 @@ export const streamNotebookRecordsAsCSV = async ( 'record_id', 'revision_id', 'type', + 'created_by', + 'created', 'updated_by', 'updated', ]; From 569ea020dbefd291d6c04e0ba0c3c37527c30137 Mon Sep 17 00:00:00 2001 From: Steve Cassidy Date: Mon, 30 Sep 2024 12:29:06 +1000 Subject: [PATCH 2/6] ensure created_by is returned in records Signed-off-by: Steve Cassidy --- library/data-model/src/data_storage/databases.ts | 3 ++- library/data-model/src/data_storage/index.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/library/data-model/src/data_storage/databases.ts b/library/data-model/src/data_storage/databases.ts index a9becb5d8..f76951812 100644 --- a/library/data-model/src/data_storage/databases.ts +++ b/library/data-model/src/data_storage/databases.ts @@ -99,8 +99,9 @@ export const addDesignDocsForNotebook = async ( if (doc.heads.length > 0) { const conflict = doc.heads.length > 1; const created = doc.created; + const created_by = doc.created_by; const type = doc.type; - emit(doc._id, {_id: doc.heads[0], conflict, created, type}); + emit(doc._id, {_id: doc.heads[0], conflict, created, created_by, type}); } }`, }, diff --git a/library/data-model/src/data_storage/index.ts b/library/data-model/src/data_storage/index.ts index 1350949db..8ae918c66 100644 --- a/library/data-model/src/data_storage/index.ts +++ b/library/data-model/src/data_storage/index.ts @@ -548,6 +548,7 @@ export async function getSomeRecords( record_id: doc.id, revision_id: doc.value._id, created: doc.value.created, + created_by: doc.value.created_by, conflict: doc.value.conflict, type: doc.value.type, revision: doc.doc, @@ -615,6 +616,7 @@ export const notebookRecordIterator = async ( } if (record) { const data = await hydrateRecord(project_id, record); + console.log('returning hydrated record', record.created_by); return {record: data, done: false}; } else { return {record: null, done: true}; From 77ef900575b44108662b7e454c97e50ef2331f6e Mon Sep 17 00:00:00 2001 From: Steve Cassidy Date: Mon, 30 Sep 2024 13:30:16 +1000 Subject: [PATCH 3/6] when there are no records to export, send just a csv with header line Signed-off-by: Steve Cassidy --- api/src/couchdb/notebooks.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/api/src/couchdb/notebooks.ts b/api/src/couchdb/notebooks.ts index 0e57db838..8b07e7d4b 100644 --- a/api/src/couchdb/notebooks.ts +++ b/api/src/couchdb/notebooks.ts @@ -52,6 +52,7 @@ import { import {userHasPermission} from './users'; PouchDB.plugin(securityPlugin); import {Stringifier, stringify} from 'csv-stringify'; +import {kStringMaxLength} from 'buffer'; /** * getProjects - get the internal project documents that reference @@ -800,7 +801,25 @@ export const streamNotebookRecordsAsCSV = async ( record = next.record; done = next.done; } - if (stringifier) stringifier.end(); + if (stringifier) { + stringifier.end(); + } else { + // no records to export so just send the bare column headings + const columns = [ + 'identifier', + 'record_id', + 'revision_id', + 'type', + 'created_by', + 'created', + 'updated_by', + 'updated', + ]; + stringifier = stringify({columns, header: true}); + // pipe output to the respose + stringifier.pipe(res); + stringifier.end(); + } }; export const streamNotebookFilesAsZip = async ( From 999bbc3c1076ca5d5bbdf9c53709cfe2ceb3fffa Mon Sep 17 00:00:00 2001 From: Steve Cassidy Date: Thu, 10 Oct 2024 12:56:01 +1100 Subject: [PATCH 4/6] remove debug output Signed-off-by: Steve Cassidy --- library/data-model/src/data_storage/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/library/data-model/src/data_storage/index.ts b/library/data-model/src/data_storage/index.ts index 8ae918c66..375a45e71 100644 --- a/library/data-model/src/data_storage/index.ts +++ b/library/data-model/src/data_storage/index.ts @@ -616,7 +616,6 @@ export const notebookRecordIterator = async ( } if (record) { const data = await hydrateRecord(project_id, record); - console.log('returning hydrated record', record.created_by); return {record: data, done: false}; } else { return {record: null, done: true}; From cc282fcecf406b67a87d408a5d04fc05d2a5d349 Mon Sep 17 00:00:00 2001 From: Steve Cassidy Date: Thu, 10 Oct 2024 13:04:41 +1100 Subject: [PATCH 5/6] Add some guards to lat/long export Signed-off-by: Steve Cassidy --- api/src/couchdb/notebooks.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/api/src/couchdb/notebooks.ts b/api/src/couchdb/notebooks.ts index 8b07e7d4b..1fdab7e62 100644 --- a/api/src/couchdb/notebooks.ts +++ b/api/src/couchdb/notebooks.ts @@ -622,7 +622,11 @@ const csvFormatValue = ( // gps locations if (fieldType === 'faims-pos::Location') { - if (value instanceof Object && 'geometry' in value) { + if ( + value instanceof Object && + 'geometry' in value && + value.geometry.coordinates.length === 2 + ) { result[fieldName] = value; result[fieldName + '_latitude'] = value.geometry.coordinates[1]; result[fieldName + '_longitude'] = value.geometry.coordinates[0]; @@ -642,7 +646,8 @@ const csvFormatValue = ( value instanceof Object && 'features' in value && value.features.length > 0 && - value.features[0]?.geometry?.type === 'Point' + value.features[0]?.geometry?.type === 'Point' && + value.features[0].geometry.coordinates.length === 2 ) { result[fieldName] = value; result[fieldName + '_latitude'] = From 9bc1f7f6d5ef287f1fa6ffcbc38f07154b35e037 Mon Sep 17 00:00:00 2001 From: Steve Cassidy Date: Thu, 10 Oct 2024 13:09:56 +1100 Subject: [PATCH 6/6] remove unused imports Signed-off-by: Steve Cassidy --- api/src/couchdb/notebooks.ts | 1 - app/src/gui/components/notebook/record_table.tsx | 1 - 2 files changed, 2 deletions(-) diff --git a/api/src/couchdb/notebooks.ts b/api/src/couchdb/notebooks.ts index 84fbb90cc..7a180b6c6 100644 --- a/api/src/couchdb/notebooks.ts +++ b/api/src/couchdb/notebooks.ts @@ -56,7 +56,6 @@ import securityPlugin from 'pouchdb-security-helper'; import {slugify} from '../utils'; import {userHasPermission} from './users'; PouchDB.plugin(securityPlugin); -import {Stringifier, stringify} from 'csv-stringify'; /** * getProjects - get the internal project documents that reference diff --git a/app/src/gui/components/notebook/record_table.tsx b/app/src/gui/components/notebook/record_table.tsx index d6cdd19da..c18f7d912 100644 --- a/app/src/gui/components/notebook/record_table.tsx +++ b/app/src/gui/components/notebook/record_table.tsx @@ -46,7 +46,6 @@ import { getMetadataForAllRecords, getRecordsWithRegex, } from '@faims3/data-model'; -import {DEBUG_APP} from '../../../buildconfig'; import {NotebookDataGridToolbar} from './datagrid_toolbar'; import RecordDelete from './delete'; import getLocalDate from '../../fields/LocalDate';