Improve the data tracing structure

Adding dataset index
and also fix bugs in PCe approaches

Author: realxinzhao

R/xfaostat_L102_ProductionArea.R

@@ -23,11 +23,11 @@ module_xfaostat_L102_ProductionArea <- function(command, ...) {
       "FBSH_CBH_wide")
 
   MODULE_OUTPUTS <-
-    c("QCL_PROD",
-      "QCL_AN_LIVEANIMAL",
-      "QCL_AN_PRIMARY_MILK",
-      "QCL_AN_LIVEANIMAL_MEATEQ",
-      "QCL_CROP_PRIMARY")
+    c("L102.QCL_PROD",
+      "L102.QCL_AN_LIVEANIMAL",
+      "L102.QCL_AN_PRIMARY_MILK",
+      "L102.QCL_AN_LIVEANIMAL_MEATEQ",
+      "L102.QCL_CROP_PRIMARY")
 
   if(command == driver.DECLARE_INPUTS) {
     return(MODULE_INPUTS)
@@ -378,7 +378,7 @@ module_xfaostat_L102_ProductionArea <- function(command, ...) {
 
 
 
-    # Bind QCL_ALL and save RDS ----
+    # Bind L102.QCL_ALL and save RDS ----
     #*******************************************
 
     QCL_CROP_PRIMARY %>% mutate(item_set = "QCL_COMM_CROP_PRIMARY") %>%
@@ -392,10 +392,10 @@ module_xfaostat_L102_ProductionArea <- function(command, ...) {
       bind_rows(QCL_AN_LIVEANIMAL %>% mutate(item_set = "QCL_COMM_AN_LIVEANIMAL")) %>%
       bind_rows(QCL_AN_LIVEANIMAL_MEATEQ %>% mutate(item_set = "QCL_COMM_AN_LIVEANIMAL_MEATEQ")) %>%
       bind_rows(FBS_FISH %>% mutate(item_set = "FBS_COMM_FISH"))->
-      QCL_ALL
+      L102.QCL_ALL
 
 
-    QCL_ALL %>%
+    L102.QCL_ALL %>%
       filter(item_set %in% c("QCL_COMM_AN_PRIMARY_MEAT1",
                              "QCL_COMM_AN_PRIMARY_MEAT2",
                              "QCL_COMM_AN_PRIMARY_EGG",
@@ -408,60 +408,60 @@ module_xfaostat_L102_ProductionArea <- function(command, ...) {
       add_units("various") %>%
       add_comments("Detailed FAO QCL data processing for live animal and production") %>%
       add_precursors("QCL_wide", "FBS_wide", "FBSH_CBH_wide") ->
-      QCL_AN_LIVEANIMAL
+      L102.QCL_AN_LIVEANIMAL
 
     QCL_AN_PRIMARY_MILK %>%
       add_title("FAO milk animal stock and production") %>%
       add_units("various") %>%
       add_comments("Detailed FAO QCL data processing for dairy animal and production") %>%
-      same_precursors_as(QCL_AN_LIVEANIMAL) ->
-      QCL_AN_PRIMARY_MILK
+      same_precursors_as(L102.QCL_AN_LIVEANIMAL) ->
+      L102.QCL_AN_PRIMARY_MILK
 
     QCL_AN_LIVEANIMAL_MEATEQ %>%
       add_title("FAO live animal stock meat equivalent") %>%
       add_units("various") %>%
       add_comments("Detailed FAO QCL data processing for live animal stock meat equivalent") %>%
-      same_precursors_as(QCL_AN_LIVEANIMAL) ->
-      QCL_AN_LIVEANIMAL_MEATEQ
+      same_precursors_as(L102.QCL_AN_LIVEANIMAL) ->
+      L102.QCL_AN_LIVEANIMAL_MEATEQ
 
     QCL_CROP_PRIMARY %>%
       add_title("FAO primary crop area and production") %>%
       add_units("various") %>%
       add_comments("Detailed FAO QCL data processing for crop area and production") %>%
-      same_precursors_as(QCL_AN_LIVEANIMAL) ->
-      QCL_CROP_PRIMARY
+      same_precursors_as(L102.QCL_AN_LIVEANIMAL) ->
+      L102.QCL_CROP_PRIMARY
 
 
     # Production only
-    QCL_ALL %>% filter(element_code == 5510) ->
-      QCL_PROD
+    L102.QCL_ALL %>% filter(element_code == 5510) ->
+      L102.QCL_PROD
 
-    QCL_PROD %>%
+    L102.QCL_PROD %>%
       add_title("FAO primary production") %>%
       add_units("tonnes") %>%
       add_comments("FAO primary production") %>%
-      same_precursors_as(QCL_AN_LIVEANIMAL) ->
-      QCL_PROD
+      same_precursors_as(L102.QCL_AN_LIVEANIMAL) ->
+      L102.QCL_PROD
 
     # No NA
-    assertthat::assert_that(QCL_ALL %>% filter(is.na(value)) %>% nrow() == 0)
+    assertthat::assert_that(L102.QCL_ALL %>% filter(is.na(value)) %>% nrow() == 0)
 
-    #QCL_ALL %>% FF_check_count_plot -> p; p
+    #L102.QCL_ALL %>% FF_check_count_plot -> p; p
     # ggsave(file.path(DIR_DATAPROC_PLOT, "QCL_ALL.png"),
     #        plot = p + ggtitle("gcamdata-FAOSTAT (QCL & FBS) production data over time"),
     #        dpi = 200, width = 9, height = 5 )
     # rm(p)
 
-    QCL_ALL %>% distinct(year); # 53 years
-    QCL_ALL %>% distinct(element, element_code, unit) # QCL_COMM_AN_LIVEANIMAL_MEATEQ has no element_code
-    QCL_ALL %>% distinct(item)  # 158 primary crop + 45 primary an + 57 others + 17 +12
+    L102.QCL_ALL %>% distinct(year); # 53 years
+    L102.QCL_ALL %>% distinct(element, element_code, unit) # QCL_COMM_AN_LIVEANIMAL_MEATEQ has no element_code
+    L102.QCL_ALL %>% distinct(item)  # 158 primary crop + 45 primary an + 57 others + 17 +12
 
 
-    # QCL_ALL %>%
+    # L102.QCL_ALL %>%
     #   add_title("FAO crop and livestock production and crop area") %>%
     #   add_units("various") %>%
     #   add_comments("Detailed FAO QCL data processing. FBS fish data is used") ->
-    #   QCL_ALL
+    #   L102.QCL_ALL
 
 
     return_data(MODULE_OUTPUTS)

R/xfaostat_L103_ProducerPrices.R

@@ -18,11 +18,11 @@
 module_xfaostat_L103_ProducerPrices <- function(command, ...) {
 
   MODULE_INPUTS <-
-    c("QCL_PROD",
+    c("L102.QCL_PROD",
       "PP_wide")
 
   MODULE_OUTPUTS <-
-    c("QCL_PRIMARY_PROD_PV")
+    c("L103.QCL_PRIMARY_PROD_PV")
 
   if(command == driver.DECLARE_INPUTS) {
     return(MODULE_INPUTS)
@@ -32,7 +32,7 @@ module_xfaostat_L103_ProducerPrices <- function(command, ...) {
 
     year <- value <- Year <- Value <- FAO_country <- iso <- NULL    # silence package check.
     PP_wide <- element_code <- element <- area_code <- item_code <- area <-
-      item <- unit <- QCL_PROD <- item_set <- GCAM_commodity <- QCL_PRIMARY_item <-
+      item <- unit <- L102.QCL_PROD <- item_set <- GCAM_commodity <- QCL_PRIMARY_item <-
       PP_item <- Production <- Prod_Value <- PP_index <- World_PP_index <-
       PP_Multiplier <- PP_item_world <- NULL
 
@@ -50,15 +50,15 @@ module_xfaostat_L103_ProducerPrices <- function(command, ...) {
 
 
     # Primary crops
-    QCL_PROD %>% filter(item_set == "QCL_COMM_CROP_PRIMARY") %>%
+    L102.QCL_PROD %>% filter(item_set == "QCL_COMM_CROP_PRIMARY") %>%
       distinct(item, item_code) -> QCL_COMM_CROP_PRIMARY
     # Primary animal products, including fat hides etc. 46
-    QCL_PROD %>% filter(grepl("AN_PRIMARY", item_set) ) %>%
+    L102.QCL_PROD %>% filter(grepl("AN_PRIMARY", item_set) ) %>%
       distinct(item, item_code) -> QCL_COMM_AN_PRIMARY
 
     QCL_COMM_CROP_PRIMARY %>%
       bind_rows(QCL_COMM_AN_PRIMARY) %>%
-      left_join(QCL_PROD, by = c("item_code", "item")) ->
+      left_join(L102.QCL_PROD, by = c("item_code", "item")) ->
       QCL_PRIMARY
 
     QCL_PRIMARY %>% distinct(item, element, item_set, unit)
@@ -129,17 +129,17 @@ module_xfaostat_L103_ProducerPrices <- function(command, ...) {
       left_join(UnitMap %>%
                   bind_rows(UnitMap %>% mutate(element = "Prod_Value", unit = "USD")),
                 by = "element") ->
-      QCL_PRIMARY_PROD_PV
+      L103.QCL_PRIMARY_PROD_PV
 
     rm(QV1, QV)
 
-    QCL_PRIMARY_PROD_PV %>%
+    L103.QCL_PRIMARY_PROD_PV %>%
       add_title("FAO crop and livestock production and crop area") %>%
       add_units("USD and tonne") %>%
       add_comments("Detailed FAO QCL data processing. FBS fish data is used") %>%
-      add_precursors("QCL_PROD",
+      add_precursors("L102.QCL_PROD",
                      "PP_wide") ->
-      QCL_PRIMARY_PROD_PV
+      L103.QCL_PRIMARY_PROD_PV
 
     return_data(MODULE_OUTPUTS)
 

R/xfaostat_L104_ProductionAreaAddFodder.R

@@ -22,7 +22,7 @@ module_xfaostat_L104_ProductionAreaAddFodder <- function(command, ...) {
       "QCL_area_code_map")
 
   MODULE_OUTPUTS <-
-    c("QCL_FODDERCROP")
+    c("L104.QCL_FODDERCROP")
 
   if(command == driver.DECLARE_INPUTS) {
     return(MODULE_INPUTS)
@@ -95,17 +95,17 @@ module_xfaostat_L104_ProductionAreaAddFodder <- function(command, ...) {
       # Remove area x year that should no exist
       FAOSTAT_AREA_RM_NONEXIST %>%
       left_join(UnitMap, by = "element") ->
-      QCL_FODDERCROP
+      L104.QCL_FODDERCROP
 
     rm(FAO_fodder_Prod_t_HA_ha_PRODSTAT_2011, UnitMap)
 
-    QCL_FODDERCROP %>%
+    L104.QCL_FODDERCROP %>%
       add_title("Processed fodder crop production and area") %>%
       add_units("tonne and ha") %>%
       add_comments("Data is from old GCAM data v5.4") %>%
       add_precursors(file.path(DIR_RAW_DATA_FAOSTAT, "Other_supplementary/FAO_fodder_Prod_t_HA_ha_PRODSTAT_2011"),
                      "QCL_area_code_map")->
-      QCL_FODDERCROP
+      L104.QCL_FODDERCROP
 
 
     # FAO_ag_items_PRODSTAT is not read in anymore