fix trailing zeros

SocialFinanceDigitalLabs · Feb 5, 2024 · 49dd939 · 49dd939
1 parent f320775
commit 49dd939
Show file tree

Hide file tree

Showing 8 changed files with 51 additions and 32 deletions.
diff --git a/liiatools/annex_a_pipeline/spec/pipeline.yml b/liiatools/annex_a_pipeline/spec/pipeline.yml
@@ -6,7 +6,7 @@ table_list:
   - id: Child Unique ID
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: Gender
     type: category
   - id: Ethnicity
@@ -32,7 +32,7 @@ table_list:
   - id: Child Unique ID
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: Gender
     type: category
   - id: Ethnicity
@@ -61,7 +61,7 @@ table_list:
   - id: Child Unique ID
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: Gender
     type: category
   - id: Ethnicity
@@ -99,7 +99,7 @@ table_list:
   - id: Child Unique ID
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: Gender
     type: category
   - id: Ethnicity
@@ -138,7 +138,7 @@ table_list:
   - id: Child Unique ID
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: Gender
     type: category
   - id: Ethnicity
@@ -181,7 +181,7 @@ table_list:
   - id: Child Unique ID
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: Gender
     type: category
   - id: Ethnicity
@@ -224,7 +224,7 @@ table_list:
   - id: Child Unique ID
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: Gender
     type: category
   - id: Ethnicity
@@ -273,7 +273,7 @@ table_list:
   - id: Child Unique ID
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: Gender
     type: category
   - id: Ethnicity
@@ -361,7 +361,7 @@ table_list:
   - id: Child Unique ID
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: Gender
     type: category
   - id: Ethnicity
@@ -399,7 +399,7 @@ table_list:
   - id: Child Unique ID
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: Family identifier
     type: string
   - id: Gender

diff --git a/liiatools/cin_census_pipeline/spec/pipeline.yml b/liiatools/cin_census_pipeline/spec/pipeline.yml
@@ -6,6 +6,7 @@ table_list:
   - id: LAchildID
     type: string
     unique_key: true
+    enrich: [integer, add_la_suffix]
   - id: Date
     type: date
     unique_key: true

diff --git a/liiatools/common/_transform_functions.py b/liiatools/common/_transform_functions.py
@@ -52,13 +52,21 @@ def add_school_year(row: pd.Series, column_config: ColumnConfig, metadata: Metad
     return school_year
 
 
+def to_integer(row: pd.Series, column_config: ColumnConfig, metadata: Metadata) -> str | int:
+    try:
+        return int(float(row[column_config.id]))
+    except ValueError:
+        return row[column_config.id]
+
+
 enrich_functions = {
     "add_la_suffix": add_la_suffix,
     "la_code": add_la_code,
     "la_name": add_la_name,
     "year": add_year,
     "quarter": add_quarter,
     "school_year": add_school_year,
+    "integer": to_integer,
 }
 
 

diff --git a/liiatools/common/data/__config.py b/liiatools/common/data/__config.py
@@ -9,7 +9,7 @@ class ColumnConfig(BaseModel):
     id: str
     type: str
     unique_key: bool = False
-    enrich: str = None
+    enrich: str | list = None
     degrade: str = None
     sort: int = None
     exclude: List[str] = []

diff --git a/liiatools/common/transform.py b/liiatools/common/transform.py
@@ -28,13 +28,23 @@ def _transform(
     for column_config in table_config.columns:
         transform_name = getattr(column_config, property)
         if transform_name:
-            assert (
-                transform_name in functions
-            ), f"Unknown transform for property '{property}': {transform_name}"
-            data[column_config.id] = data.apply(
-                lambda row: functions[transform_name](row, column_config, metadata),
-                axis=1,
-            )
+            if isinstance(transform_name, list):
+                for transform_name in transform_name:
+                    assert (
+                        transform_name in functions
+                    ), f"Unknown transform for property '{property}': {transform_name}"
+                    data[column_config.id] = data.apply(
+                        lambda row: functions[transform_name](row, column_config, metadata),
+                        axis=1,
+                    )
+            else:
+                assert (
+                    transform_name in functions
+                ), f"Unknown transform for property '{property}': {transform_name}"
+                data[column_config.id] = data.apply(
+                    lambda row: functions[transform_name](row, column_config, metadata),
+                    axis=1,
+                )
 
 
 def data_transforms(

diff --git a/liiatools/s251_pipeline/spec/pipeline.yml b/liiatools/s251_pipeline/spec/pipeline.yml
@@ -6,7 +6,7 @@ table_list:
   - id: Child ID
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: Date of birth
     type: date
     degrade: first_of_month
@@ -85,7 +85,7 @@ table_list:
   - id: Asset name
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: Ofsted URN
     type: string
   - id: Start date

diff --git a/liiatools/ssda903_pipeline/spec/pipeline.yml b/liiatools/ssda903_pipeline/spec/pipeline.yml
@@ -4,7 +4,7 @@ table_list:
   - id: CHILD
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: DOB
     type: date
     degrade: first_of_month
@@ -37,7 +37,7 @@ table_list:
   - id: CHILD
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: DECOM
     type: date
     unique_key: true
@@ -81,7 +81,7 @@ table_list:
   - id: CHILD
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: SEX
     type: category
   - id: DOB
@@ -117,7 +117,7 @@ table_list:
   - id: CHILD
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: DOB
     type: date
     degrade: first_of_month
@@ -144,7 +144,7 @@ table_list:
   - id: CHILD
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: DOB
     type: date
     unique_key: true
@@ -203,7 +203,7 @@ table_list:
   - id: CHILD
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: DOB
     type: date
     unique_key: true
@@ -230,7 +230,7 @@ table_list:
   columns:
   - id: CHILD
     type: string
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: DOB
     type: date
     degrade: first_of_month
@@ -256,7 +256,7 @@ table_list:
   - id: CHILD
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: DOB
     type: date
     degrade: first_of_month
@@ -281,7 +281,7 @@ table_list:
   - id: CHILD
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: DOB
     type: date
     degrade: first_of_month
@@ -305,7 +305,7 @@ table_list:
   - id: CHILD
     type: string
     unique_key: true
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: DOB
     type: date
     degrade: first_of_month
@@ -325,7 +325,7 @@ table_list:
   columns:
   - id: CHILD
     type: string
-    enrich: add_la_suffix
+    enrich: [integer, add_la_suffix]
   - id: DOB
     type: date
     degrade: first_of_month

diff --git a/tests/cin_census/test_reports.py b/tests/cin_census/test_reports.py
@@ -191,7 +191,7 @@ def test_s47_journeys():
             "S47ActualStartDate",
             "CPPstartDate",
             "DateOfInitialCPC",
-            "YEAR",
+            "Year",
         ],
     )
-Original file line number
+Diff line change
@@ Expand Up / @@ -191,7 +191,7 @@ def test_s47_journeys(): @@
                 "S47ActualStartDate",
                 "CPPstartDate",
                 "DateOfInitialCPC",
-                "YEAR",
+                "Year",
             ],
         )
@@ Expand Down @@