add CIN csv pipeline

liiatools/cin_census_pipeline/pipeline.py

@@ -15,10 +15,14 @@
 
 from liiatools.cin_census_pipeline.spec import (
     load_pipeline_config,
-    load_schema,
+    load_xml_schema,
+    load_csv_schema,
     load_schema_path,
 )
-from liiatools.cin_census_pipeline.stream_pipeline import task_cleanfile
+from liiatools.cin_census_pipeline.stream_pipeline import (
+    task_xml_cleanfile,
+    task_csv_cleanfile,
+)
 from liiatools.cin_census_pipeline.reports import reports
 
 
@@ -55,13 +59,21 @@ def process_file(
     uuid = file_locator.meta["uuid"]
 
     # Load schema and set on processing metadata
-    schema = load_schema(year=year)
-    schema_path = load_schema_path(year=year)
+    if file_locator.name.endswith(".xml"):
+        schema = load_xml_schema(year=year)
+        schema_path = load_schema_path(year=year)
+    elif file_locator.name.endswith(".csv"):
+        schema = load_csv_schema(year=year)
+
     metadata = dict(year=year, schema=schema, la_code=la_code)
 
     # Normalise the data and export to the session 'cleaned' folder
     try:
-        cleanfile_result = task_cleanfile(file_locator, schema, schema_path)
+        cleanfile_result = (
+            task_xml_cleanfile(file_locator, schema, schema_path)
+            if file_locator.name.endswith(".xml")
+            else task_csv_cleanfile(file_locator, schema)
+        )
     except Exception as e:
         logger.exception(f"Error cleaning file {file_locator.name}")
         errors.append(
@@ -155,19 +167,19 @@ def process_session(source_fs: FS, output_fs: FS, la_code: str):
         report_data.data.export(report_folder, "cin_census_", "csv")
 
     # Run report analysis
-    analysis_data = report_data.data["CIN"]
-
-    expanded_assessment_factors = reports.expanded_assessment_factors(analysis_data)
-    referral_outcomes = reports.referral_outcomes(analysis_data)
-    s47_journeys = reports.s47_journeys(analysis_data)
-
-    analysis_data = DataContainer(
-        {
-            "factors": expanded_assessment_factors,
-            "referrals": referral_outcomes,
-            "S47_journeys": s47_journeys,
-        }
-    )
-
-    analysis_folder = export_folder.makedirs("REPORTS", recreate=True)
-    analysis_data.export(analysis_folder, "cin_census_", "csv")
+    # analysis_data = report_data.data["CIN"]
+    #
+    # expanded_assessment_factors = reports.expanded_assessment_factors(analysis_data)
+    # referral_outcomes = reports.referral_outcomes(analysis_data)
+    # s47_journeys = reports.s47_journeys(analysis_data)
+    #
+    # analysis_data = DataContainer(
+    #     {
+    #         "factors": expanded_assessment_factors,
+    #         "referrals": referral_outcomes,
+    #         "S47_journeys": s47_journeys,
+    #     }
+    # )
+    #
+    # analysis_folder = export_folder.makedirs("REPORTS", recreate=True)
+    # analysis_data.export(analysis_folder, "cin_census_", "csv")

liiatools/cin_census_pipeline/spec/CIN_schema_2016.yml

@@ -38,7 +38,7 @@ column_map:
         upn:
             &upn
             string: "regex"
-            cell_regex: "[A-Za-z]\d{11}(\d|[A-Za-z])"
+            cell_regex: "[A-Za-z]\\d{11}(\\d|[A-Za-z])"
             canbeblank: yes
         formerupn:
             *upn
@@ -128,7 +128,6 @@ column_map:
         cppenddate:
             *date-blank
         numberofpreviouscpp:
-            &int-not-blank
             numeric:
               type: "integer"
               min_value: 0

liiatools/cin_census_pipeline/spec/CIN_schema_2017.diff.yml

@@ -0,0 +1,5 @@
+column_map.childprotectionplans:
+  type: remove
+  description: Remove column from childprotectionplans
+  value:
+    - seensocialworker

liiatools/cin_census_pipeline/spec/CIN_schema_2021.diff.yml

@@ -0,0 +1,87 @@
+column_map.factorsidentifiedatassessment.assessmentfactors:
+  type: modify
+  description: Updated assessmentfactors codes for 2021 schema
+  value:
+    category:
+    - code: "1A"
+      name: "Alcohol misuse: concerns about alcohol misuse by the child"
+    - code: "1B"
+      name: "Alcohol misuse: concerns about alcohol misuse by the parent(s)/carer(s)"
+    - code: "1C"
+      name: "Alcohol misuse: concerns about alcohol misuse by another person living in the household"
+    - code: "2A"
+      name: "Drug misuse: concerns about drug misuse by the child"
+    - code: "2B"
+      name: "Drug misuse: concerns about drug misuse by the parent(s)/carer(s)"
+    - code: "2C"
+      name: "Drug misuse: concerns about drug misuse by another person living in the household"
+    - code: "3A"
+      name: "Domestic violence: concerns about the child being the subject of domestic violence"
+    - code: "3B"
+      name: "Domestic violence: concerns about the child’s parent(s)/carer(s) being the subject of domestic violence"
+    - code: "3C"
+      name: "Domestic violence: concerns about another person living in the household being the subject of domestic violence"
+    - code: "4A"
+      name: "Mental health: concerns about the mental health of the child"
+    - code: "4B"
+      name: "Mental health: concerns about the mental health of the parent(s)/carer(s)"
+    - code: "4C"
+      name: "Mental health: concerns about the mental health of another person in the family/household"
+    - code: "5A"
+      name: "Learning disability: concerns about the child’s learning disability"
+    - code: "5B"
+      name: "Learning disability: concerns about the parent(s)/carer(s) learning disability"
+    - code: "5C"
+      name: "Learning disability: concerns about another person in the family/household’s learning disability"
+    - code: "6A"
+      name: "Physical disability or illness: concerns about a physical disability or illness of the child"
+    - code: "6B"
+      name: "Physical disability or illness: concerns about a physical disability or illness of the parent(s)/carer(s)"
+    - code: "6C"
+      name: "Physical disability or illness: concerns about a physical disability or illness of another person in the family/household"
+    - code: "7A"
+      name: "Young carer: concerns that services may be required or the child’s health or development may be impaired due to their caring responsibilities"
+    - code: "8B"
+      name: "Privately fostered: concerns that services may be required or the child may be at risk as a privately fostered child - overseas children who intend to return"
+    - code: "8C"
+      name: "Privately fostered: concerns that services may be required or the child may be at risk as a privately fostered child - overseas children who intend to stay"
+    - code: "8D"
+      name: "Privately fostered: concerns that services may be required or the child may be at risk as a privately fostered child - UK children in educational placements"
+    - code: "8E"
+      name: "Privately fostered: concerns that services may be required or the child may be at risk as a privately fostered child - UK children making alternative family arrangements"
+    - code: "8F"
+      name: "Privately fostered: concerns that services may be required or the child may be at risk as a privately fostered child - other"
+    - code: "9A"
+      name: "UASC: concerns that services may be required or the child may be at risk of harm as an unaccompanied asylum-seeking child"
+    - code: "10A"
+      name: "Missing: concerns that services may be required or the child may be at risk of harm due to going/being missing"
+    - code: "11A"
+      name: "Child sexual exploitation: concerns that services may be required or the child may be at risk of harm due to child sexual exploitation"
+    - code: "12A"
+      name: "Trafficking: concerns that services may be required or the child may be at risk of harm due to trafficking"
+    - code: "13A"
+      name: "Gangs: concerns that services may be required or the child may be at risk of harm because of involvement in/with gangs"
+    - code: "14A"
+      name: "Socially unacceptable behaviour: concerns that services may be required or the child may be at risk due to their socially unacceptable behaviour"
+    - code: "15A"
+      name: "Self-harm: concerns that services may be required or due to suspected/actual self-harming child may be at risk of harm"
+    - code: "16A"
+      name: "Abuse or neglect – ‘NEGLECT’: concerns that services may be required or the child may be suffering or likely to suffer significant harm due to abuse or neglect"
+    - code: "17A"
+      name: "Abuse or neglect – ‘EMOTIONAL ABUSE’: concerns that services may be required or the child may be suffering or likely to suffer significant harm due to abuse or neglect"
+    - code: "18B"
+      name: "Abuse or neglect – ‘PHYSICAL ABUSE’ (child on child): concerns that services may be required or the child may be suffering or likely to suffer significant harm due to abuse or neglect by another child"
+    - code: "18C"
+      name: "Abuse or neglect – ‘PHYSICAL ABUSE’ (adult on child): concerns that services may be required or the child may be suffering or likely to suffer significant harm due to abuse or neglect by an adult"
+    - code: "19B"
+      name: "Abuse or neglect – ‘SEXUAL ABUSE’ (child on child): concerns that services may be required or the child may be suffering or likely to suffer significant harm due to abuse or neglect by another child"
+    - code: "19C"
+      name: "Abuse or neglect – ‘SEXUAL ABUSE’ (adult on child): concerns that services may be required or the child may be suffering or likely to suffer significant harm due to abuse or neglect by an adult"
+    - code: "20"
+      name: "Other"
+    - code: "21"
+      name: "No factors identified - only use this if there is no evidence of any of the factors above and no further action is being taken"
+    - code: "22A"
+      name: "Female genital mutilation (FGM) - concerns that services may be required or the child may be at risk due to female genital mutilation"
+    - code: "23A"
+      name: "Abuse linked to faith or belief - concerns that services may be required or the child may be at risk due to abuse linked to faith or belief"

liiatools/cin_census_pipeline/spec/CIN_schema_2022.diff.yml

@@ -0,0 +1,89 @@
+column_map.factorsidentifiedatassessment.assessmentfactors:
+  type: modify
+  description: Updated assessmentfactors codes for 2022 schema
+  value:
+    category:
+    - code: "1A"
+      name: "Alcohol misuse: concerns about alcohol misuse by the child"
+    - code: "1B"
+      name: "Alcohol misuse: concerns about alcohol misuse by the parent(s)/carer(s)"
+    - code: "1C"
+      name: "Alcohol misuse: concerns about alcohol misuse by another person living in the household"
+    - code: "2A"
+      name: "Drug misuse: concerns about drug misuse by the child"
+    - code: "2B"
+      name: "Drug misuse: concerns about drug misuse by the parent(s)/carer(s)"
+    - code: "2C"
+      name: "Drug misuse: concerns about drug misuse by another person living in the household"
+    - code: "3A"
+      name: "Domestic violence: concerns about the child being the subject of domestic violence"
+    - code: "3B"
+      name: "Domestic violence: concerns about the child’s parent(s)/carer(s) being the subject of domestic violence"
+    - code: "3C"
+      name: "Domestic violence: concerns about another person living in the household being the subject of domestic violence"
+    - code: "4A"
+      name: "Mental health: concerns about the mental health of the child"
+    - code: "4B"
+      name: "Mental health: concerns about the mental health of the parent(s)/carer(s)"
+    - code: "4C"
+      name: "Mental health: concerns about the mental health of another person in the family/household"
+    - code: "5A"
+      name: "Learning disability: concerns about the child’s learning disability"
+    - code: "5B"
+      name: "Learning disability: concerns about the parent(s)/carer(s) learning disability"
+    - code: "5C"
+      name: "Learning disability: concerns about another person in the family/household’s learning disability"
+    - code: "6A"
+      name: "Physical disability or illness: concerns about a physical disability or illness of the child"
+    - code: "6B"
+      name: "Physical disability or illness: concerns about a physical disability or illness of the parent(s)/carer(s)"
+    - code: "6C"
+      name: "Physical disability or illness: concerns about a physical disability or illness of another person in the family/household"
+    - code: "7A"
+      name: "Young carer: concerns that services may be required or the child’s health or development may be impaired due to their caring responsibilities"
+    - code: "8B"
+      name: "Privately fostered: concerns that services may be required or the child may be at risk as a privately fostered child - overseas children who intend to return"
+    - code: "8C"
+      name: "Privately fostered: concerns that services may be required or the child may be at risk as a privately fostered child - overseas children who intend to stay"
+    - code: "8D"
+      name: "Privately fostered: concerns that services may be required or the child may be at risk as a privately fostered child - UK children in educational placements"
+    - code: "8E"
+      name: "Privately fostered: concerns that services may be required or the child may be at risk as a privately fostered child - UK children making alternative family arrangements"
+    - code: "8F"
+      name: "Privately fostered: concerns that services may be required or the child may be at risk as a privately fostered child - other"
+    - code: "9A"
+      name: "UASC: concerns that services may be required or the child may be at risk of harm as an unaccompanied asylum-seeking child"
+    - code: "10A"
+      name: "Missing: concerns that services may be required or the child may be at risk of harm due to going/being missing"
+    - code: "11A"
+      name: "Child sexual exploitation: concerns that services may be required or the child may be at risk of harm due to child sexual exploitation"
+    - code: "12A"
+      name: "Trafficking: concerns that services may be required or the child may be at risk of harm due to trafficking"
+    - code: "13A"
+      name: "Gangs: concerns that services may be required or the child may be at risk of harm because of involvement in/with gangs"
+    - code: "14A"
+      name: "Socially unacceptable behaviour: concerns that services may be required or the child may be at risk due to their socially unacceptable behaviour"
+    - code: "15A"
+      name: "Self-harm: concerns that services may be required or due to suspected/actual self-harming child may be at risk of harm"
+    - code: "16A"
+      name: "Abuse or neglect – ‘NEGLECT’: concerns that services may be required or the child may be suffering or likely to suffer significant harm due to abuse or neglect"
+    - code: "17A"
+      name: "Abuse or neglect – ‘EMOTIONAL ABUSE’: concerns that services may be required or the child may be suffering or likely to suffer significant harm due to abuse or neglect"
+    - code: "18B"
+      name: "Abuse or neglect – ‘PHYSICAL ABUSE’ (child on child): concerns that services may be required or the child may be suffering or likely to suffer significant harm due to abuse or neglect by another child"
+    - code: "18C"
+      name: "Abuse or neglect – ‘PHYSICAL ABUSE’ (adult on child): concerns that services may be required or the child may be suffering or likely to suffer significant harm due to abuse or neglect by an adult"
+    - code: "19B"
+      name: "Abuse or neglect – ‘SEXUAL ABUSE’ (child on child): concerns that services may be required or the child may be suffering or likely to suffer significant harm due to abuse or neglect by another child"
+    - code: "19C"
+      name: "Abuse or neglect – ‘SEXUAL ABUSE’ (adult on child): concerns that services may be required or the child may be suffering or likely to suffer significant harm due to abuse or neglect by an adult"
+    - code: "20"
+      name: "Other"
+    - code: "21"
+      name: "No factors identified - only use this if there is no evidence of any of the factors above and no further action is being taken"
+    - code: "22A"
+      name: "Female genital mutilation (FGM) - concerns that services may be required or the child may be at risk due to female genital mutilation"
+    - code: "23A"
+      name: "Abuse linked to faith or belief - concerns that services may be required or the child may be at risk due to abuse linked to faith or belief"
+    - code: "24A"
+      name: "Child criminal exploitation: concerns that services may be required or the child may be at risk of harm due to child criminal exploitation"

liiatools/cin_census_pipeline/spec/CIN_schema_2023.diff.yml

@@ -0,0 +1,23 @@
+column_map.cindetails.reasonforclosure:
+  type: modify
+  description: Updated reasonforclosure codes for 2023 schema
+  value:
+    category:
+    - code: "RC1"
+      name: "Adopted"
+    - code: "RC2"
+      name: "Died"
+    - code: "RC3"
+      name: "Child arrangements order"
+    - code: "RC4"
+      name: "Special guardianship order"
+    - code: "RC5"
+      name: "Transferred to services of another local authority"
+    - code: "RC6"
+      name: "Transferred to adult social care services"
+    - code: "RC7"
+      name: "Services ceased for any other reason, including child no longer in need"
+    - code: "RC8"
+      name: "Case closed after assessment, no further action"
+    - code: "RC9"
+      name: "Case closed after assessment, referred to early help"