fix metadata encoding issues with æøå

KasperSkytte · Dec 14, 2023 · 63b53b2 · 63b53b2
1 parent 15cd3ce
commit 63b53b2
Show file tree

Hide file tree

Showing 25 changed files with 94 additions and 89 deletions.
diff --git a/data/create_datasets.Rmd b/data/create_datasets.Rmd
@@ -10,6 +10,10 @@ library(ampvis2)
 library(openxlsx)
 library(data.table)
 library(tidyverse)
+# need at least data.table version 1.14.99 to allow writing files with encoding
+if (packageVersion("data.table") < "1.14.99") {
+  stop("data.table must be version 1.14.99 or later. Upgrade with \"data.table::update_dev_pkg()\" or \"install.packages('data.table', repos='https://rdatatable.gitlab.io/data.table', ref = '1.14.99')\" if it's not available on CRAN for your R version.")
+}
 
 if (interactive()) {
   if (!grepl("data$", getwd())) {
@@ -182,7 +186,6 @@ datasets <- c(
 
 # Select and create data subsets
 ```{r export_datasets, eval = TRUE}
-
 for (wwtp in datasets) {
   #filter
   dataset <- amp_subset_samples(
@@ -213,6 +216,8 @@ for (wwtp in datasets) {
   #write out metadata
   fwrite(
     dataset$metadata,
+    bom = TRUE,
+    encoding = "UTF-8",
     file = paste0(dataset_folder, "/metadata.csv")
   )
 }

diff --git a/data/datasets/Aalborg E/metadata.csv b/data/datasets/Aalborg E/metadata.csv
@@ -1,4 +1,4 @@
-Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
+Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
 MQ191115-203,MQ191115,LIB-Aalborg-E-LT-580,79589,Aalborg oest #580,Summer,LT-2,"",Aalborg E,2018-08-08,32,2018,,"","","","","","",,,,,,,,,20.4
 MQ191115-204,MQ191115,LIB-Aalborg-E-LT-582,79885,Aalborg oest #582,Summer,LT-1,"",Aalborg E,2018-08-15,33,2018,,"","","","","","",,,,,,,,,18.3
 MQ191115-205,MQ191115,LIB-Aalborg-E-LT-584,74007,Aalborg oest #584,Summer,LT-2,"",Aalborg E,2018-08-21,34,2018,,"","","","","","",,,,,,,,,18.5

diff --git a/data/datasets/Aalborg W/metadata.csv b/data/datasets/Aalborg W/metadata.csv
@@ -1,4 +1,4 @@
-Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
+Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
 MQ181221-245,MQ181221,LIB-Aalborg-Vest-LT-379,,"",Winter,Activated sludge,"",Aalborg W,2017-02-07,6,2017,,"","","","","","",,,,,,,,,9.4
 MQ181221-246,MQ181221,LIB-Aalborg-Vest-LT-382,,"",Winter,Activated sludge,"",Aalborg W,2017-02-15,7,2017,,"","","","","","",,,,,,,,,10.7
 MQ181221-247,MQ181221,LIB-Aalborg-Vest-LT-385,,"",Winter,Activated sludge,"",Aalborg W,2017-02-21,8,2017,,"","","","","","",,,,,,,,,

diff --git a/data/datasets/Avedøre/metadata.csv b/data/datasets/Avedøre/metadata.csv
@@ -1,4 +1,4 @@
-Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
+Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
 MQ210618-102,MQ210618,LIB-Avedoere-LT-96,,"",Fall,"","",Avedøre,2020-09-18,38,2020,,"","","","","","",,,,,,,,,
 MQ210618-103,MQ210618,LIB-Avedoere-LT-98,,"",Fall,"","",Avedøre,2020-10-01,40,2020,,"","","","","","",,,,,,,,,
 MQ210618-104,MQ210618,LIB-Avedoere-LT-100,,"",Fall,"","",Avedøre,2020-10-19,43,2020,,"","","","","","",,,,,,,,,
@@ -86,26 +86,26 @@ MQ201019-271,MQ201019,LIB-Avedore-LT-84,55611,"",Summer,"","",Avedøre,2020-07-2
 MQ201019-272,MQ201019,LIB-Avedore-LT-86,52880,"",Summer,"","",Avedøre,2020-08-04,32,2020,,"","","","","","",,,,,,,,,
 MQ201019-273,MQ201019,LIB-Avedore-LT-88,56322,"",Summer,"","",Avedøre,2020-08-19,34,2020,,"","","","","","",,,,,,,,,
 MQ201019-274,MQ201019,LIB-Avedore-LT-92,56181,"",Summer,"","",Avedøre,2020-08-24,35,2020,,"","","","","","",,,,,,,,,
-MQ220601-197,MQ220601,LIB-Aved??�re-LT-137,,"",Spring,"",Activated sludge,Avedøre,2021-04-08,14,2021,,"","","","","","",,,,,,,,,
-MQ220601-198,MQ220601,LIB-Aved??�re-LT-139,,"",Spring,"",Activated sludge,Avedøre,2021-04-20,16,2021,,"","","","","","",,,,,,,,,
-MQ220601-199,MQ220601,LIB-Aved??�re-LT-147,,"",Spring,"",Activated sludge,Avedøre,2021-04-26,17,2021,,"","","","","","",,,,,,,,,
-MQ220601-200,MQ220601,LIB-Aved??�re-LT-149,,"",Spring,"",Activated sludge,Avedøre,2021-05-10,19,2021,,"","","","","","",,,,,,,,,
-MQ220601-201,MQ220601,LIB-Aved??�re-LT-151,,"",Spring,"",Activated sludge,Avedøre,2021-05-19,20,2021,,"","","","","","",,,,,,,,,
-MQ220601-202,MQ220601,LIB-Aved??�re-LT-159,,"",Spring,"",Activated sludge,Avedøre,2021-06-09,23,2021,,"","","","","","",,,,,,,,,
-MQ220601-203,MQ220601,LIB-Aved??�re-LT-163,,"",Summer,"",Activated sludge,Avedøre,2021-06-16,24,2021,,"","","","","","",,,,,,,,,
-MQ220601-204,MQ220601,LIB-Aved??�re-LT-171,,"",Summer,"",Activated sludge,Avedøre,2021-06-21,25,2021,,"","","","","","",,,,,,,,,
-MQ220601-205,MQ220601,LIB-Aved??�re-LT-173,,"",Summer,"",Activated sludge,Avedøre,2021-07-06,27,2021,,"","","","","","",,,,,,,,,
-MQ220601-206,MQ220601,LIB-Aved??�re-LT-175,,"",Summer,"",Activated sludge,Avedøre,2021-07-15,28,2021,,"","","","","","",,,,,,,,,
-MQ220601-207,MQ220601,LIB-Aved??�re-LT-183,,"",Summer,"",Activated sludge,Avedøre,2021-07-21,29,2021,,"","","","","","",,,,,,,,,
-MQ220601-208,MQ220601,LIB-Aved??�re-LT-185,,"",Fall,"",Activated sludge,Avedøre,2021-09-15,37,2021,,"","","","","","",,,,,,,,,
-MQ220601-209,MQ220601,LIB-Aved??�re-LT-193,,"",Fall,"",Activated sludge,Avedøre,2021-09-17,37,2021,,"","","","","","",,,,,,,,,
-MQ220601-210,MQ220601,LIB-Aved??�re-LT-195,,"",Fall,"",Activated sludge,Avedøre,2021-09-30,39,2021,,"","","","","","",,,,,,,,,
-MQ220601-211,MQ220601,LIB-Aved??�re-LT-197,,"",Fall,"",Activated sludge,Avedøre,2021-10-07,40,2021,,"","","","","","",,,,,,,,,
-MQ220601-212,MQ220601,LIB-Aved??�re-LT-199,,"",Fall,"",Activated sludge,Avedøre,2021-10-18,42,2021,,"","","","","","",,,,,,,,,
-MQ220601-213,MQ220601,LIB-Aved??�re-LT-207,,"",Fall,"",Activated sludge,Avedøre,2021-10-25,43,2021,,"","","","","","",,,,,,,,,
-MQ220601-214,MQ220601,LIB-Aved??�re-LT-209,,"",Fall,"",Activated sludge,Avedøre,2021-11-16,46,2021,,"","","","","","",,,,,,,,,
-MQ220601-215,MQ220601,LIB-Aved??�re-LT-211,,"",Fall,"",Activated sludge,Avedøre,2021-11-19,46,2021,,"","","","","","",,,,,,,,,
-MQ220601-216,MQ220601,LIB-Aved??�re-LT-219,,"",Fall,"",Activated sludge,Avedøre,2021-11-29,48,2021,,"","","","","","",,,,,,,,,
-MQ220601-217,MQ220601,LIB-Aved??�re-LT-221,,"",Fall,"",Activated sludge,Avedøre,2021-12-13,50,2021,,"","","","","","",,,,,,,,,
-MQ220601-218,MQ220601,LIB-Aved??�re-LT-223,,"",Winter,"",Activated sludge,Avedøre,2021-12-16,50,2021,,"","","","","","",,,,,,,,,
-MQ220601-219,MQ220601,LIB-Aved??�re-LT-233,,"",Winter,"",Activated sludge,Avedøre,2022-01-18,3,2022,,"","","","","","",,,,,,,,,
+MQ220601-197,MQ220601,LIB-Aved??óre-LT-137,,"",Spring,"",Activated sludge,Avedøre,2021-04-08,14,2021,,"","","","","","",,,,,,,,,
+MQ220601-198,MQ220601,LIB-Aved??óre-LT-139,,"",Spring,"",Activated sludge,Avedøre,2021-04-20,16,2021,,"","","","","","",,,,,,,,,
+MQ220601-199,MQ220601,LIB-Aved??óre-LT-147,,"",Spring,"",Activated sludge,Avedøre,2021-04-26,17,2021,,"","","","","","",,,,,,,,,
+MQ220601-200,MQ220601,LIB-Aved??óre-LT-149,,"",Spring,"",Activated sludge,Avedøre,2021-05-10,19,2021,,"","","","","","",,,,,,,,,
+MQ220601-201,MQ220601,LIB-Aved??óre-LT-151,,"",Spring,"",Activated sludge,Avedøre,2021-05-19,20,2021,,"","","","","","",,,,,,,,,
+MQ220601-202,MQ220601,LIB-Aved??óre-LT-159,,"",Spring,"",Activated sludge,Avedøre,2021-06-09,23,2021,,"","","","","","",,,,,,,,,
+MQ220601-203,MQ220601,LIB-Aved??óre-LT-163,,"",Summer,"",Activated sludge,Avedøre,2021-06-16,24,2021,,"","","","","","",,,,,,,,,
+MQ220601-204,MQ220601,LIB-Aved??óre-LT-171,,"",Summer,"",Activated sludge,Avedøre,2021-06-21,25,2021,,"","","","","","",,,,,,,,,
+MQ220601-205,MQ220601,LIB-Aved??óre-LT-173,,"",Summer,"",Activated sludge,Avedøre,2021-07-06,27,2021,,"","","","","","",,,,,,,,,
+MQ220601-206,MQ220601,LIB-Aved??óre-LT-175,,"",Summer,"",Activated sludge,Avedøre,2021-07-15,28,2021,,"","","","","","",,,,,,,,,
+MQ220601-207,MQ220601,LIB-Aved??óre-LT-183,,"",Summer,"",Activated sludge,Avedøre,2021-07-21,29,2021,,"","","","","","",,,,,,,,,
+MQ220601-208,MQ220601,LIB-Aved??óre-LT-185,,"",Fall,"",Activated sludge,Avedøre,2021-09-15,37,2021,,"","","","","","",,,,,,,,,
+MQ220601-209,MQ220601,LIB-Aved??óre-LT-193,,"",Fall,"",Activated sludge,Avedøre,2021-09-17,37,2021,,"","","","","","",,,,,,,,,
+MQ220601-210,MQ220601,LIB-Aved??óre-LT-195,,"",Fall,"",Activated sludge,Avedøre,2021-09-30,39,2021,,"","","","","","",,,,,,,,,
+MQ220601-211,MQ220601,LIB-Aved??óre-LT-197,,"",Fall,"",Activated sludge,Avedøre,2021-10-07,40,2021,,"","","","","","",,,,,,,,,
+MQ220601-212,MQ220601,LIB-Aved??óre-LT-199,,"",Fall,"",Activated sludge,Avedøre,2021-10-18,42,2021,,"","","","","","",,,,,,,,,
+MQ220601-213,MQ220601,LIB-Aved??óre-LT-207,,"",Fall,"",Activated sludge,Avedøre,2021-10-25,43,2021,,"","","","","","",,,,,,,,,
+MQ220601-214,MQ220601,LIB-Aved??óre-LT-209,,"",Fall,"",Activated sludge,Avedøre,2021-11-16,46,2021,,"","","","","","",,,,,,,,,
+MQ220601-215,MQ220601,LIB-Aved??óre-LT-211,,"",Fall,"",Activated sludge,Avedøre,2021-11-19,46,2021,,"","","","","","",,,,,,,,,
+MQ220601-216,MQ220601,LIB-Aved??óre-LT-219,,"",Fall,"",Activated sludge,Avedøre,2021-11-29,48,2021,,"","","","","","",,,,,,,,,
+MQ220601-217,MQ220601,LIB-Aved??óre-LT-221,,"",Fall,"",Activated sludge,Avedøre,2021-12-13,50,2021,,"","","","","","",,,,,,,,,
+MQ220601-218,MQ220601,LIB-Aved??óre-LT-223,,"",Winter,"",Activated sludge,Avedøre,2021-12-16,50,2021,,"","","","","","",,,,,,,,,
+MQ220601-219,MQ220601,LIB-Aved??óre-LT-233,,"",Winter,"",Activated sludge,Avedøre,2022-01-18,3,2022,,"","","","","","",,,,,,,,,
diff --git a/data/datasets/Damhusåen-A/metadata.csv b/data/datasets/Damhusåen-A/metadata.csv
@@ -1,4 +1,4 @@
-Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
+Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
 MQ190919-39,MQ190919,LIB-Damhusaen-LT-A-159,,"",Spring,A,mean,Damhusåen-A,2019-04-11,15,2019,,"","","","","","",,,,,,,,,
 MQ190919-40,MQ190919,LIB-Damhusaen-LT-A-160,,"",Spring,A,mean,Damhusåen-A,2019-04-15,16,2019,,"","","","","","",,,,,,,,,
 MQ190919-41,MQ190919,LIB-Damhusaen-LT-A-161,,"",Spring,A,mean,Damhusåen-A,2019-04-24,17,2019,,"","","","","","",,,,,,,,,

diff --git a/data/datasets/Damhusåen-B/metadata.csv b/data/datasets/Damhusåen-B/metadata.csv
@@ -1,4 +1,4 @@
-Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
+Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
 MQ190919-96,MQ190919,LIB-Damhusaen-LT-B-161,,"",Spring,B,mean,Damhusåen-B,2019-04-11,15,2019,,"","","","","","",,,,,,,,,
 MQ190919-97,MQ190919,LIB-Damhusaen-LT-B-162,,"",Spring,B,mean,Damhusåen-B,2019-04-15,16,2019,,"","","","","","",,,,,,,,,
 MQ190919-99,MQ190919,LIB-Damhusaen-LT-B-164,,"",Spring,B,mean,Damhusåen-B,2019-04-25,17,2019,,"","","","","","",,,,,,,,,

diff --git a/data/datasets/Damhusåen-C/metadata.csv b/data/datasets/Damhusåen-C/metadata.csv
@@ -1,4 +1,4 @@
-Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
+Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
 MQ190919-201,MQ190919,LIB-Damhusaen-LT-C-160,,"",Spring,C,mean,Damhusåen-C,2019-04-11,15,2019,,"","","","","","",,,,,,,,,
 MQ190919-202,MQ190919,LIB-Damhusaen-LT-C-161,,"",Spring,C,mean,Damhusåen-C,2019-04-15,16,2019,,"","","","","","",,,,,,,,,
 MQ190919-203,MQ190919,LIB-Damhusaen-LT-C-162,,"",Spring,C,mean,Damhusåen-C,2019-04-24,17,2019,,"","","","","","",,,,,,,,,

diff --git a/data/datasets/Damhusåen-D/metadata.csv b/data/datasets/Damhusåen-D/metadata.csv
@@ -1,4 +1,4 @@
-Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
+Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
 MQ190919-255,MQ190919,LIB-Damhusaen-LT-D-160,,"",Spring,D,mean,Damhusåen-D,2019-04-09,15,2019,,"","","","","","",,,,,,,,,
 MQ190919-256,MQ190919,LIB-Damhusaen-LT-D-161,,"",Spring,D,mean,Damhusåen-D,2019-04-11,15,2019,,"","","","","","",,,,,,,,,
 MQ190919-257,MQ190919,LIB-Damhusaen-LT-D-162,,"",Spring,D,mean,Damhusåen-D,2019-04-15,16,2019,,"","","","","","",,,,,,,,,

diff --git a/data/datasets/Egå/metadata.csv b/data/datasets/Egå/metadata.csv
@@ -1,4 +1,4 @@
-Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
+Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
 MQ181123-181,MQ181123,LIB-Egaa-LT-1,,Egaa #1,Summer,"","",Egå,2016-08-31,35,2016,,Recirculation,"","","","","",,,,,,,28.01775981,2.008973583,
 MQ181123-182,MQ181123,LIB-Egaa-LT-4,,Egaa #4,Summer,"","",Egå,2016-09-09,36,2016,,Recirculation,"","","","","",,,,,,,19.0751593,3.425323905,
 MQ181123-183,MQ181123,LIB-Egaa-LT-7,,Egaa #7,Fall,"","",Egå,2016-09-16,37,2016,,Recirculation,"","","","","",,,,,,,19.39632009,2.643443595,