add + process the remaining data sets

KasperSkytte · Dec 13, 2023 · 15cd3ce · 15cd3ce
1 parent 7640d72
commit 15cd3ce
Show file tree

Hide file tree

Showing 47 changed files with 25,975 additions and 13 deletions.
diff --git a/data/create_datasets.Rmd b/data/create_datasets.Rmd
@@ -10,7 +10,6 @@ library(ampvis2)
 library(openxlsx)
 library(data.table)
 library(tidyverse)
-library(doFuture)
 
 if (interactive()) {
   if (!grepl("data$", getwd())) {
@@ -183,15 +182,12 @@ datasets <- c(
 
 # Select and create data subsets
 ```{r export_datasets, eval = TRUE}
-# speed it up a bit by running in parallel (have enough memory!!)
-options("future.globals.maxSize" = 10*1024^3) #increase to 10GB
-future::plan(multisession, workers = 12)
 
-dlist <- foreach(wwtp = datasets, export = ds) %dofuture% {
+for (wwtp in datasets) {
   #filter
   dataset <- amp_subset_samples(
     ds,
-    Plant %chin% wwtp,
+    Plant %in% wwtp,
     normalise = FALSE
   ) %>%
     filter_otus(0.1)
@@ -219,8 +215,6 @@ dlist <- foreach(wwtp = datasets, export = ds) %dofuture% {
     dataset$metadata,
     file = paste0(dataset_folder, "/metadata.csv")
   )
-
-  dataset
 }
 
 ```
@@ -260,7 +254,6 @@ samples_overview_all <- ggplot(
     axis.title = element_blank(),
     axis.text.x = element_text(angle = 90))
 
-samples_overview_all
 ggsave("samples_overview_all.png", plot = samples_overview_all)
 ```
 
@@ -272,7 +265,6 @@ samples_overview <- ggplot(m[used == "used"], aes(Date, Plant)) +
   theme(
     axis.title = element_blank(),
     axis.text.x = element_text(angle = 90))
-samples_overview
 ggsave("samples_overview_used.png", plot = samples_overview)
 ```
 
@@ -290,6 +282,5 @@ sampling_intervals <- ggplot(
   theme(axis.title.y = element_blank()) +
   xlab("Sampling interval (days)") +
   scale_x_continuous(trans = "sqrt", breaks = c(1,7,14,21,28,90))
-sampling_intervals
 ggsave("sampling_intervals.png", plot = sampling_intervals)
 ```
diff --git a/data/create_datasets_files/figure-markdown_strict/samples_overview-1.png b/data/create_datasets_files/figure-markdown_strict/samples_overview-1.png
diff --git a/data/create_datasets_files/figure-markdown_strict/samples_overview_all-1.png b/data/create_datasets_files/figure-markdown_strict/samples_overview_all-1.png
diff --git a/data/create_datasets_files/figure-markdown_strict/sampling_intervals-1.png b/data/create_datasets_files/figure-markdown_strict/sampling_intervals-1.png
diff --git a/data/datasets/Egå/ASVtable.csv b/data/datasets/Egå/ASVtable.csv
diff --git a/data/datasets/Egå/metadata.csv b/data/datasets/Egå/metadata.csv
diff --git a/data/datasets/Egå/taxonomy.csv b/data/datasets/Egå/taxonomy.csv
diff --git a/data/datasets/Esbjerg E/ASVtable.csv b/data/datasets/Esbjerg E/ASVtable.csv
diff --git a/data/datasets/Esbjerg E/metadata.csv b/data/datasets/Esbjerg E/metadata.csv
diff --git a/data/datasets/Esbjerg E/taxonomy.csv b/data/datasets/Esbjerg E/taxonomy.csv
diff --git a/data/datasets/Esbjerg W/ASVtable.csv b/data/datasets/Esbjerg W/ASVtable.csv
diff --git a/data/datasets/Esbjerg W/metadata.csv b/data/datasets/Esbjerg W/metadata.csv
diff --git a/data/datasets/Esbjerg W/taxonomy.csv b/data/datasets/Esbjerg W/taxonomy.csv
diff --git a/data/datasets/Fredericia/ASVtable.csv b/data/datasets/Fredericia/ASVtable.csv
diff --git a/data/datasets/Fredericia/metadata.csv b/data/datasets/Fredericia/metadata.csv
diff --git a/data/datasets/Fredericia/taxonomy.csv b/data/datasets/Fredericia/taxonomy.csv
diff --git a/data/datasets/Haderslev/ASVtable.csv b/data/datasets/Haderslev/ASVtable.csv
diff --git a/data/datasets/Haderslev/metadata.csv b/data/datasets/Haderslev/metadata.csv
diff --git a/data/datasets/Haderslev/taxonomy.csv b/data/datasets/Haderslev/taxonomy.csv
diff --git a/data/datasets/Hirtshals/ASVtable.csv b/data/datasets/Hirtshals/ASVtable.csv
diff --git a/data/datasets/Hirtshals/metadata.csv b/data/datasets/Hirtshals/metadata.csv
diff --git a/data/datasets/Hirtshals/taxonomy.csv b/data/datasets/Hirtshals/taxonomy.csv
diff --git a/data/datasets/Hjørring/ASVtable.csv b/data/datasets/Hjørring/ASVtable.csv
diff --git a/data/datasets/Hjørring/metadata.csv b/data/datasets/Hjørring/metadata.csv
diff --git a/data/datasets/Hjørring/taxonomy.csv b/data/datasets/Hjørring/taxonomy.csv
diff --git a/data/datasets/Kalundborg/ASVtable.csv b/data/datasets/Kalundborg/ASVtable.csv
diff --git a/data/datasets/Kalundborg/metadata.csv b/data/datasets/Kalundborg/metadata.csv
diff --git a/data/datasets/Kalundborg/taxonomy.csv b/data/datasets/Kalundborg/taxonomy.csv
diff --git a/data/datasets/Lynetten/ASVtable.csv b/data/datasets/Lynetten/ASVtable.csv
diff --git a/data/datasets/Lynetten/metadata.csv b/data/datasets/Lynetten/metadata.csv
@@ -0,0 +1,93 @@
+Sample,SeqRun,LibID,Read no,ID,Period,Line,Mean,Plant,Date,Week,Year,Period2,Configuration,Design,Design2,ReturnSludgeSidestream,PrimarySetling,Digester,lat,long,Solids_AerationTank_PointMeasurement,SolidsFraction_AerationTank_PointMeasurement,SVI,DSVI,ExtractionConc,LibraryConc,week_mean_temperature
+MQ210618-86,MQ210618,LIB-Lynetten-LT-82,,"",Summer,"","",Lynetten,2020-09-02,36,2020,,"","","","","","",,,,,,,,,
+MQ210618-87,MQ210618,LIB-Lynetten-LT-84,,"",Fall,"","",Lynetten,2020-09-16,38,2020,,"","","","","","",,,,,,,,,
+MQ210618-88,MQ210618,LIB-Lynetten-LT-86,,"",Fall,"","",Lynetten,2020-10-08,41,2020,,"","","","","","",,,,,,,,,
+MQ210618-89,MQ210618,LIB-Lynetten-LT-88,,"",Fall,"","",Lynetten,2020-11-04,45,2020,,"","","","","","",,,,,,,,,
+MQ210618-90,MQ210618,LIB-Lynetten-LT-90,,"",Fall,"","",Lynetten,2020-11-24,48,2020,,"","","","","","",,,,,,,,,
+MQ210618-91,MQ210618,LIB-Lynetten-LT-92,,"",Fall,"","",Lynetten,2020-12-08,50,2020,,"","","","","","",,,,,,,,,
+MQ210618-92,MQ210618,LIB-Lynetten-LT-94,,"",Winter,"","",Lynetten,2021-01-07,1,2021,,"","","","","","",,,,,,,,,
+MQ210618-93,MQ210618,LIB-Lynetten-LT-96,,"",Winter,"","",Lynetten,2021-01-25,4,2021,,"","","","","","",,,,,,,,,
+MQ210618-94,MQ210618,LIB-Lynetten-LT-98,,"",Winter,"","",Lynetten,2021-02-02,5,2021,,"","","","","","",,,,,,,,,
+MQ210618-95,MQ210618,LIB-Lynetten-LT-100,,"",Winter,"","",Lynetten,2021-02-16,7,2021,,"","","","","","",,,,,,,,,
+MQ210618-96,MQ210618,LIB-Lynetten-LT-102,,"",Winter,"","",Lynetten,2021-03-02,9,2021,,"","","","","","",,,,,,,,,
+MQ210618-97,MQ210618,LIB-Lynetten-LT-104,,"",Spring,"","",Lynetten,2021-03-16,11,2021,,"","","","","","",,,,,,,,,
+MQ181023-129,MQ181023,LIB-BB-LYNETTEN-20171020,,LYNETTEN-20171020,Fall,"","",Lynetten,2017-10-20,42,2017,,"","","","","","",,,,,,,17.78036778,6.146039982,
+MQ181023-130,MQ181023,LIB-BB-LYNETTEN-20171109,,LYNETTEN-20171109,Fall,"","",Lynetten,2017-11-09,45,2017,,"","","","","","",,,,,,,16.66019166,7.959458772,
+MQ181023-131,MQ181023,LIB-BB-LYNETTEN-20171130,,LYNETTEN-20171130,Fall,"","",Lynetten,2017-11-30,48,2017,,"","","","","","",,,,,,,22.60813261,9.059463859,
+MQ181023-132,MQ181023,LIB-BB-LYNETTEN-20171208,,LYNETTEN-20171208,Fall,"","",Lynetten,2017-12-08,49,2017,,"","","","","","",,,,,,,20.15281015,11.1831731,
+MQ181023-133,MQ181023,LIB-BB-LYNETTEN-20171221,,LYNETTEN-20171221,Winter,"","",Lynetten,2017-12-21,51,2017,,"","","","","","",,,,,,,19.02356902,8.262119131,
+MQ181023-135,MQ181023,LIB-BB-LYNETTEN-20180124,,LYNETTEN-20180124,Winter,"","",Lynetten,2018-01-24,4,2018,,"","","","","","",,,,,,,35.82491582,0.389134747,
+MQ181023-136,MQ181023,LIB-BB-LYNETTEN-20180208,,LYNETTEN-20180208,Winter,"","",Lynetten,2018-02-08,6,2018,,"","","","","","",,,,,,,33.64154364,3.766722621,
+MQ181023-137,MQ181023,LIB-BB-LYNETTEN-20180223,,LYNETTEN-20180223,Winter,"","",Lynetten,2018-02-23,8,2018,,"","","","","","",,,,,,,36.13312613,7.533445241,
+MQ181023-138,MQ181023,LIB-BB-LYNETTEN-20180226,,LYNETTEN-20180226,Winter,"","",Lynetten,2018-02-26,9,2018,,"","","","","","",,,,,,,31.44781145,13.93890839,
+MQ181023-139,MQ181023,LIB-BB-LYNETTEN-20180309,,LYNETTEN-20180309,Winter,"","",Lynetten,2018-03-09,10,2018,,"","","","","","",,,,,,,37.66770267,14.64977873,
+MQ181023-140,MQ181023,LIB-BB-LYNETTEN-20180318,,LYNETTEN-20180318,Spring,"","",Lynetten,2018-03-18,11,2018,,"","","","","","",,,,,,,27.67029267,10.90085966,
+MQ181023-141,MQ181023,LIB-BB-LYNETTEN-20180323,,LYNETTEN-20180323,Spring,"","",Lynetten,2018-03-23,12,2018,,"","","","","","",,,,,,,33.08469308,9.972531665,
+MQ181023-142,MQ181023,LIB-BB-LYNETTEN-20180416,,LYNETTEN-20180416,Spring,"","",Lynetten,2018-04-16,16,2018,,"","","","","","",,,,,,,27.28826729,9.119232921,
+MQ181023-143,MQ181023,LIB-BB-LYNETTEN-20180430,,LYNETTEN-20180430,Spring,"","",Lynetten,2018-04-30,18,2018,,"","","","","","",,,,,,,20.64361564,7.017142276,
+MQ181023-144,MQ181023,LIB-BB-LYNETTEN-20180529,,LYNETTEN-20180529,Spring,"","",Lynetten,2018-05-29,22,2018,,"","","","","","",,,,,,,20.15281015,4.859097614,
+MQ181023-145,MQ181023,LIB-BB-LYNETTEN-20180606,,LYNETTEN-20180606,Spring,"","",Lynetten,2018-06-06,23,2018,,"","","","","","",,,,,,,17.29344729,6.691591637,
+MQ181023-146,MQ181023,LIB-BB-LYNETTEN-20180705,,LYNETTEN-20180705,Summer,"","",Lynetten,2018-07-05,27,2018,,"","","","","","",,,,,,,17.57057757,6.349509131,
+MQ181023-147,MQ181023,LIB-BB-LYNETTEN-20180801,,LYNETTEN-20180801,Summer,"","",Lynetten,2018-08-01,31,2018,,"","","","","","",,,,,,,6.052836053,8.1858182,
+MQ190919-113,MQ190919,LIB-Lynetten-LT-44,,"",Summer,"","",Lynetten,2018-08-18,33,2018,,"","","","","","",,,,,,,,,
+MQ190919-114,MQ190919,LIB-Lynetten-LT-45,,"",Summer,"","",Lynetten,2018-08-28,35,2018,,"","","","","","",,,,,,,,,
+MQ190919-115,MQ190919,LIB-Lynetten-LT-46,,"",Summer,"","",Lynetten,2018-09-11,37,2018,,"","","","","","",,,,,,,,,
+MQ190919-116,MQ190919,LIB-Lynetten-LT-47,,"",Fall,"","",Lynetten,2018-09-24,39,2018,,"","","","","","",,,,,,,,,
+MQ190919-117,MQ190919,LIB-Lynetten-LT-48,,"",Fall,"","",Lynetten,2018-10-11,41,2018,,"","","","","","",,,,,,,,,
+MQ190919-118,MQ190919,LIB-Lynetten-LT-49,,"",Fall,"","",Lynetten,2018-10-26,43,2018,,"","","","","","",,,,,,,,,
+MQ190919-119,MQ190919,LIB-Lynetten-LT-50,,"",Fall,"","",Lynetten,2018-11-15,46,2018,,"","","","","","",,,,,,,,,
+MQ190919-120,MQ190919,LIB-Lynetten-LT-51,,"",Fall,"","",Lynetten,2018-11-27,48,2018,,"","","","","","",,,,,,,,,
+MQ190919-121,MQ190919,LIB-Lynetten-LT-52,,"",Fall,"","",Lynetten,2018-12-04,49,2018,,"","","","","","",,,,,,,,,
+MQ190919-122,MQ190919,LIB-Lynetten-LT-53,,"",Winter,"","",Lynetten,2018-12-20,51,2018,,"","","","","","",,,,,,,,,
+MQ190919-123,MQ190919,LIB-Lynetten-LT-54,,"",Winter,"","",Lynetten,2019-02-25,9,2019,,"","","","","","",,,,,,,,,
+MQ190919-125,MQ190919,LIB-Lynetten-LT-56,,"",Spring,"","",Lynetten,2019-04-15,16,2019,,"","","","","","",,,,,,,,,
+MQ190919-126,MQ190919,LIB-Lynetten-LT-57,,"",Spring,"","",Lynetten,2019-04-30,18,2019,,"","","","","","",,,,,,,,,
+MQ190919-127,MQ190919,LIB-Lynette-LT-1,,"",Spring,"","",Lynetten,2019-05-09,19,2019,,"","","","","","",,,,,,,,,
+MQ190919-128,MQ190919,LIB-Lynetten-LT-3,,"",Spring,"","",Lynetten,2019-05-20,21,2019,,"","","","","","",,,,,,,,,
+MQ190919-129,MQ190919,LIB-Lynetten-LT-5,,"",Spring,"","",Lynetten,2019-06-13,24,2019,,"","","","","","",,,,,,,,,
+MQ190919-130,MQ190919,LIB-Lynetten-LT-7,,"",Summer,"","",Lynetten,2019-06-27,26,2019,,"","","","","","",,,,,,,,,
+MQ190919-131,MQ190919,LIB-Lynetten-LT-9,,"",Summer,"","",Lynetten,2019-07-10,28,2019,,"","","","","","",,,,,,,,,
+MQ201009-274,MQ201009,LIB-Lynetten-LT-11,38122,"",Summer,"","",Lynetten,2019-07-26,30,2019,,"","","","","","",,,,,,,,,
+MQ201009-275,MQ201009,LIB-Lynetten-LT-13,36863,"",Summer,"","",Lynetten,2019-08-08,32,2019,,"","","","","","",,,,,,,,,
+MQ201009-276,MQ201009,LIB-Lynetten-LT-15,35679,"",Summer,"","",Lynetten,2019-08-29,35,2019,,"","","","","","",,,,,,,,,
+MQ201009-277,MQ201009,LIB-Lynetten-LT-17,39487,"",Fall,"","",Lynetten,2019-09-19,38,2019,,"","","","","","",,,,,,,,,
+MQ201009-278,MQ201009,LIB-Lynetten-LT-19,39924,"",Fall,"","",Lynetten,2019-10-17,42,2019,,"","","","","","",,,,,,,,,
+MQ201009-279,MQ201009,LIB-Lynetten-LT-21,38476,"",Fall,"","",Lynetten,2019-11-14,46,2019,,"","","","","","",,,,,,,,,
+MQ201009-280,MQ201009,LIB-Lynetten-LT-22,39247,"",Fall,"","",Lynetten,2019-11-27,48,2019,,"","","","","","",,,,,,,,,
+MQ201009-281,MQ201009,LIB-Lynetten-LT-24,38213,"",Fall,"","",Lynetten,2019-12-12,50,2019,,"","","","","","",,,,,,,,,
+MQ201009-282,MQ201009,LIB-Lynetten-LT-26,1313,"",Winter,"","",Lynetten,2019-12-18,51,2019,,"","","","","","",,,,,,,,,
+MQ201009-283,MQ201009,LIB-Lynetten-LT-28,36868,"",Winter,"","",Lynetten,2020-01-09,2,2020,,"","","","","","",,,,,,,,,
+MQ201009-284,MQ201009,LIB-Lynetten-LT-30,33971,"",Winter,"","",Lynetten,2020-01-15,3,2020,,"","","","","","",,,,,,,,,
+MQ201009-285,MQ201009,LIB-Lynetten-LT-32,36693,"",Winter,"","",Lynetten,2020-02-11,7,2020,,"","","","","","",,,,,,,,,
+MQ201009-286,MQ201009,LIB-Lynetten-LT-34,35891,"",Winter,"","",Lynetten,2020-02-25,9,2020,,"","","","","","",,,,,,,,,
+MQ201009-287,MQ201009,LIB-Lynetten-LT-36,38876,"",Winter,"","",Lynetten,2020-03-11,11,2020,,"","","","","","",,,,,,,,,
+MQ201009-288,MQ201009,LIB-Lynetten-LT-38,41065,"",Spring,"","",Lynetten,2020-05-14,20,2020,,"","","","","","",,,,,,,,,
+MQ201009-289,MQ201009,LIB-Lynetten-LT-40,39107,"",Spring,"","",Lynetten,2020-05-28,22,2020,,"","","","","","",,,,,,,,,
+MQ201009-290,MQ201009,LIB-Lynetten-LT-42,39486,"",Spring,"","",Lynetten,2020-06-10,24,2020,,"","","","","","",,,,,,,,,
+MQ201009-291,MQ201009,LIB-Lynetten-LT-44,35855,"",Summer,"","",Lynetten,2020-06-23,26,2020,,"","","","","","",,,,,,,,,
+MQ201009-292,MQ201009,LIB-Lynetten-LT-46,35949,"",Summer,"","",Lynetten,2020-07-09,28,2020,,"","","","","","",,,,,,,,,
+MQ201009-293,MQ201009,LIB-Lynetten-LT-48,40063,"",Summer,"","",Lynetten,2020-07-23,30,2020,,"","","","","","",,,,,,,,,
+MQ201009-294,MQ201009,LIB-Lynetten-LT-51,38521,"",Summer,"","",Lynetten,2020-08-14,33,2020,,"","","","","","",,,,,,,,,
+MQ201009-295,MQ201009,LIB-Lynetten-LT-53,43322,"",Summer,"","",Lynetten,2020-08-28,35,2020,,"","","","","","",,,,,,,,,
+MQ220601-254,MQ220601,LIB-Lynetten-LT-106,,"",Spring,"",Activated sludge,Lynetten,2021-04-06,14,2021,,"","","","","","",,,,,,,,,
+MQ220601-255,MQ220601,LIB-Lynetten-LT-108,,"",Spring,"",Activated sludge,Lynetten,2021-04-21,16,2021,,"","","","","","",,,,,,,,,
+MQ220601-256,MQ220601,LIB-Lynetten-LT-110,,"",Spring,"",Activated sludge,Lynetten,2021-05-04,18,2021,,"","","","","","",,,,,,,,,
+MQ220601-257,MQ220601,LIB-Lynetten-LT-112,,"",Spring,"",Activated sludge,Lynetten,2021-05-18,20,2021,,"","","","","","",,,,,,,,,
+MQ220601-258,MQ220601,LIB-Lynetten-LT-114,,"",Spring,"",Activated sludge,Lynetten,2021-06-09,23,2021,,"","","","","","",,,,,,,,,
+MQ220601-259,MQ220601,LIB-Lynetten-LT-116,,"",Summer,"",Activated sludge,Lynetten,2021-06-24,25,2021,,"","","","","","",,,,,,,,,
+MQ220601-260,MQ220601,LIB-Lynetten-LT-118,,"",Summer,"",Activated sludge,Lynetten,2021-07-05,27,2021,,"","","","","","",,,,,,,,,
+MQ220601-261,MQ220601,LIB-Lynetten-LT-120,,"",Summer,"",Activated sludge,Lynetten,2021-07-14,28,2021,,"","","","","","",,,,,,,,,
+MQ220601-262,MQ220601,LIB-Lynetten-LT-122,,"",Summer,"",Activated sludge,Lynetten,2021-07-29,30,2021,,"","","","","","",,,,,,,,,
+MQ220601-263,MQ220601,LIB-Lynetten-LT-126,,"",Summer,"",Activated sludge,Lynetten,2021-08-25,34,2021,,"","","","","","",,,,,,,,,
+MQ220601-264,MQ220601,LIB-Lynetten-LT-128,,"",Summer,"",Activated sludge,Lynetten,2021-09-14,37,2021,,"","","","","","",,,,,,,,,
+MQ220601-265,MQ220601,LIB-Lynetten-LT-130,,"",Fall,"",Activated sludge,Lynetten,2021-09-27,39,2021,,"","","","","","",,,,,,,,,
+MQ220601-266,MQ220601,LIB-Lynetten-LT-132,,"",Fall,"",Activated sludge,Lynetten,2021-10-07,40,2021,,"","","","","","",,,,,,,,,
+MQ220601-267,MQ220601,LIB-Lynetten-LT-134,,"",Fall,"",Activated sludge,Lynetten,2021-10-19,42,2021,,"","","","","","",,,,,,,,,
+MQ220601-268,MQ220601,LIB-Lynetten-LT-136,,"",Fall,"",Activated sludge,Lynetten,2021-11-04,44,2021,,"","","","","","",,,,,,,,,
+MQ220601-269,MQ220601,LIB-Lynetten-LT-138,,"",Fall,"",Activated sludge,Lynetten,2021-11-11,45,2021,,"","","","","","",,,,,,,,,
+MQ220601-270,MQ220601,LIB-Lynetten-LT-140,,"",Fall,"",Activated sludge,Lynetten,2021-11-23,47,2021,,"","","","","","",,,,,,,,,
+MQ220601-271,MQ220601,LIB-Lynetten-LT-142,,"",Fall,"",Activated sludge,Lynetten,2021-12-09,49,2021,,"","","","","","",,,,,,,,,
+MQ220601-272,MQ220601,LIB-Lynetten-LT-144,,"",Winter,"",Activated sludge,Lynetten,2021-12-21,51,2021,,"","","","","","",,,,,,,,,
+MQ220601-273,MQ220601,LIB-Lynetten-LT-146,,"",Winter,"",Activated sludge,Lynetten,2022-01-06,1,2022,,"","","","","","",,,,,,,,,
+MQ220601-274,MQ220601,LIB-Lynetten-LT-148,,"",Winter,"",Activated sludge,Lynetten,2022-01-21,3,2022,,"","","","","","",,,,,,,,,
+MQ220601-275,MQ220601,LIB-Lynetten-LT-150,,"",Winter,"",Activated sludge,Lynetten,2022-01-25,4,2022,,"","","","","","",,,,,,,,,