Updated Scotland RTT; refactored code

R/data.R

@@ -416,7 +416,7 @@ NULL
 #'
 #' A dataset containing RTT data for Scotland.
 #'
-#' @format A data frame with 144 rows and 5 variables:
+#' @format A data frame with 153 rows and 5 variables:
 #' \describe{
 #'   \item{date}{Date}
 #'   \item{year}{Year}
@@ -431,7 +431,7 @@ NULL
 #'
 #' A dataset containing RTT data for Scottish Health Boards.
 #'
-#' @format A data frame with 2,160 rows and 7 variables:
+#' @format A data frame with 2,295 rows and 7 variables:
 #' \describe{
 #'   \item{hb19_code}{Scottish Health Board code}
 #'   \item{date}{Date}

data-raw/england-rtt.R

@@ -21,7 +21,6 @@ nhs_region_lookup <- tribble(
   "NHS ENGLAND SOUTH WEST (SOUTH WEST NORTH)", "South West",
   "NHS ENGLAND SOUTH EAST (HAMPSHIRE, ISLE OF WIGHT AND THAMES VALLEY)", "South East",
   "NHS ENGLAND SOUTH EAST (KENT, SURREY AND SUSSEX)", "South East",
-
   "NHS ENGLAND NORTH (YORKSHIRE AND HUMBER)", "North East and Yorkshire",
   "NHS ENGLAND NORTH (CUMBRIA AND NORTH EAST)", "North East and Yorkshire",
   "NHS ENGLAND NORTH (CHESHIRE AND MERSEYSIDE)", "North West",
@@ -44,7 +43,6 @@ urls <- c(
   mar_23 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2023/05/Full-CSV-data-file-Mar23-ZIP-3823K-53773.zip",
   feb_23 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2023/04/Full-CSV-data-file-Feb23-ZIP-3552K-55444.zip",
   jan_23 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2023/03/Full-CSV-data-file-Jan23-ZIP-3608K-03732.zip",
-
   dec_22 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2023/02/Full-CSV-data-file-Dec22-ZIP-3407K-58481.zip",
   nov_22 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2023/01/Full-CSV-data-file-Nov22-ZIP-3510K-63230.zip",
   oct_22 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2022/12/Full-CSV-data-file-Oct22-ZIP-3701K-v2.zip",
@@ -57,7 +55,6 @@ urls <- c(
   mar_22 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2023/03/Full-CSV-data-file-Mar22-revised-ZIP-111805K.zip",
   feb_22 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2023/03/Full-CSV-data-file-Feb22-revised-ZIP-109268K.zip",
   jan_22 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2023/03/Full-CSV-data-file-Jan22-revised-ZIP-4266K.zip",
-
   dec_21 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2023/03/Full-CSV-data-file-Dec21-revised-ZIP-3744K.zip",
   nov_21 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2023/03/Full-CSV-data-file-Nov21-revised-ZIP-3826K.zip",
   oct_21 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2023/03/Full-CSV-data-file-Oct21-revised-ZIP-3787K.zip",
@@ -70,7 +67,6 @@ urls <- c(
   mar_21 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2021/05/Full-CSV-data-file-Mar21-ZIP-2888K-76325.zip",
   feb_21 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2021/04/Full-CSV-data-file-Feb21-ZIP-2739K-25692.zip",
   jan_21 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2021/03/Full-CSV-data-file-Jan21-ZIP-2714K-24158.zip",
-
   dec_20 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2021/10/Full-CSV-data-file-Dec20-revised-ZIP-2860K.zip",
   nov_20 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2021/01/Full-CSV-data-file-Nov20-ZIP-2758K-26885.zip",
   oct_20 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2021/10/Full-CSV-data-file-Oct20-revised-ZIP-2772K.zip",
@@ -83,7 +79,6 @@ urls <- c(
   mar_20 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2020/05/Full-CSV-data-file-Mar20-ZIP-2995K-73640.zip",
   feb_20 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2022/01/Full-CSV-data-file-Feb20-revised-ZIP-3171K.zip",
   jan_20 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2022/01/Full-CSV-data-file-Jan20-revised-ZIP-3227K.zip",
-
   dec_19 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2022/01/Full-CSV-data-file-Dec19-revised-ZIP-3109K.zip",
   nov_19 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2022/01/Full-CSV-data-file-Nov19-revised-ZIP-3528K.zip",
   oct_19 = "https://www.england.nhs.uk/statistics/wp-content/uploads/sites/2/2022/01/Full-CSV-data-file-Oct19-revised-ZIP-3580K.zip",
@@ -111,7 +106,7 @@ for (url in urls) {
 
 # Sustainability Transformation Partnerships and NHS England (Region) (April 2020) Lookup in England
 # Source: https://geoportal.statistics.gov.uk/datasets/ons::sustainability-transformation-partnerships-and-nhs-england-region-april-2020-lookup-in-england-1/about
-stp_region <- read_sf("https://services1.arcgis.com/ESMARspQHYMw9BZ9/arcgis/rest/services/STP20_NHSER20_EN_LU_e268d9cb3626464584f6b988a0aa4e61/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson") |> 
+stp_region <- read_sf("https://services1.arcgis.com/ESMARspQHYMw9BZ9/arcgis/rest/services/STP20_NHSER20_EN_LU_e268d9cb3626464584f6b988a0aa4e61/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson") |>
   st_drop_geometry()
 
 # ---- Load waiting list data into separate dataframes ----
@@ -129,9 +124,11 @@ for (file in list.files(td, pattern = "*.csv", full.names = TRUE)) {
   # Add date columns
   d <-
     d %>%
-    mutate(Date = dmy(str_replace(Period, "RTT", "01")),
-           Month = month.abb[month(Date)],
-           Year = year(Date))
+    mutate(
+      Date = dmy(str_replace(Period, "RTT", "01")),
+      Month = month.abb[month(Date)],
+      Year = year(Date)
+    )
 
   # Data from April 2021 onward contains STPs/ICSs
   if (d$Date[1] >= dmy("01-04-2021")) {
@@ -142,10 +139,11 @@ for (file in list.files(td, pattern = "*.csv", full.names = TRUE)) {
         `Total waiting > 18 weeks` = rowSums(across(`Gt 18 To 19 Weeks SUM 1`:`Gt 104 Weeks SUM 1`), na.rm = TRUE),
         `Total waiting > 52 weeks` = rowSums(across(`Gt 52 To 53 Weeks SUM 1`:`Gt 104 Weeks SUM 1`), na.rm = TRUE)
       ) %>%
-
       group_by(Year, Month, `Provider Parent Org Code`, `Provider Parent Name`, `Treatment Function Name`) %>%
-      summarise(`Total waiting > 52 weeks` = sum(`Total waiting > 52 weeks`, na.rm = TRUE),
-                `Total waiting > 18 weeks` = sum(`Total waiting > 18 weeks`, na.rm = TRUE))
+      summarise(
+        `Total waiting > 52 weeks` = sum(`Total waiting > 52 weeks`, na.rm = TRUE),
+        `Total waiting > 18 weeks` = sum(`Total waiting > 18 weeks`, na.rm = TRUE)
+      )
 
     d_flow <-
       d %>%
@@ -156,11 +154,9 @@ for (file in list.files(td, pattern = "*.csv", full.names = TRUE)) {
         str_detect(`RTT Part Description`, "^Incomplete") ~ "Incomplete",
         str_detect(`RTT Part Description`, "^New") ~ "New"
       )) %>%
-
       group_by(Year, Month, `Provider Parent Org Code`, Pathway) %>%
       summarise(`Total All` = sum(`Total All`, na.rm = TRUE)) %>%
       ungroup() %>%
-
       pivot_wider(names_from = Pathway, values_from = `Total All`)
 
     # Bind to main STP dataframe
@@ -171,16 +167,16 @@ for (file in list.files(td, pattern = "*.csv", full.names = TRUE)) {
     d <-
       d %>%
       left_join(stp_region, by = c("Provider Parent Org Code" = "STP20CDH"))
-
   } else if (d$Date[1] >= dmy("01-04-2020")) {
     # Calculate STP/ICS totals
     d_stp <-
       d %>%
       mutate(`Total waiting > 18 weeks` = rowSums(across(`Gt 18 To 19 Weeks SUM 1`:`Gt 52 Weeks SUM 1`), na.rm = TRUE)) %>%
-
       group_by(Year, Month, `Provider Parent Org Code`, `Provider Parent Name`, `Treatment Function Name`) %>%
-      summarise(`Total waiting > 52 weeks` = sum(`Gt 52 Weeks SUM 1`, na.rm = TRUE),
-                `Total waiting > 18 weeks` = sum(`Total waiting > 18 weeks`, na.rm = TRUE))
+      summarise(
+        `Total waiting > 52 weeks` = sum(`Gt 52 Weeks SUM 1`, na.rm = TRUE),
+        `Total waiting > 18 weeks` = sum(`Total waiting > 18 weeks`, na.rm = TRUE)
+      )
 
     d_flow <-
       d %>%
@@ -191,11 +187,9 @@ for (file in list.files(td, pattern = "*.csv", full.names = TRUE)) {
         str_detect(`RTT Part Description`, "^Incomplete") ~ "Incomplete",
         str_detect(`RTT Part Description`, "^New") ~ "New"
       )) %>%
-
       group_by(Year, Month, `Provider Parent Org Code`, Pathway) %>%
       summarise(`Total All` = sum(`Total All`, na.rm = TRUE)) %>%
       ungroup() %>%
-
       pivot_wider(names_from = Pathway, values_from = `Total All`)
 
     # Bind to main STP dataframe
@@ -206,9 +200,7 @@ for (file in list.files(td, pattern = "*.csv", full.names = TRUE)) {
     d <-
       d %>%
       left_join(stp_region, by = c("Provider Parent Org Code" = "STP20CDH"))
-
   } else {
-
     # Data before April 2020 already contains NHS Regions, so use lookup table at the top of this script to sanitise the names
     d <-
       d %>%
@@ -220,28 +212,26 @@ for (file in list.files(td, pattern = "*.csv", full.names = TRUE)) {
   }
 
   if (d$Date[1] >= dmy("01-04-2021")) {
-
     d_region <-
       d %>%
       mutate(
         `Total waiting > 18 weeks` = rowSums(across(`Gt 18 To 19 Weeks SUM 1`:`Gt 104 Weeks SUM 1`), na.rm = TRUE),
         `Total waiting > 52 weeks` = rowSums(across(`Gt 52 To 53 Weeks SUM 1`:`Gt 104 Weeks SUM 1`), na.rm = TRUE)
       ) %>%
-
       group_by(Year, Month, NHSER20NM, `Treatment Function Name`) %>%
-      summarise(`Total waiting > 52 weeks` = sum(`Total waiting > 52 weeks`, na.rm = TRUE),
-                `Total waiting > 18 weeks` = sum(`Total waiting > 18 weeks`, na.rm = TRUE))
-
+      summarise(
+        `Total waiting > 52 weeks` = sum(`Total waiting > 52 weeks`, na.rm = TRUE),
+        `Total waiting > 18 weeks` = sum(`Total waiting > 18 weeks`, na.rm = TRUE)
+      )
   } else {
-
     d_region <-
       d %>%
       mutate(`Total waiting > 18 weeks` = rowSums(across(`Gt 18 To 19 Weeks SUM 1`:`Gt 52 Weeks SUM 1`), na.rm = TRUE)) %>%
-
       group_by(Year, Month, NHSER20NM, `Treatment Function Name`) %>%
-      summarise(`Total waiting > 52 weeks` = sum(`Gt 52 Weeks SUM 1`, na.rm = TRUE),
-                `Total waiting > 18 weeks` = sum(`Total waiting > 18 weeks`, na.rm = TRUE))
-
+      summarise(
+        `Total waiting > 52 weeks` = sum(`Gt 52 Weeks SUM 1`, na.rm = TRUE),
+        `Total waiting > 18 weeks` = sum(`Total waiting > 18 weeks`, na.rm = TRUE)
+      )
   }
 
   # Add regional data to main dataframe

data-raw/ni-rtt.R

@@ -1,9 +1,17 @@
 library(tidyverse)
 library(lubridate)
+library(devtools)
 
-# Inpatient & Day Case waiting times: https://www.health-ni.gov.uk/publications/northern-ireland-waiting-time-statistics-inpatient-and-day-case-waiting-times-march-2023
-# Previous Q: "https://www.health-ni.gov.uk/sites/default/files/publications/health/hs-niwts-tables-inpatient-and-day-case-waiting-q1-23-24.csv"
-ni_inpatient <- read_csv("https://www.health-ni.gov.uk/sites/default/files/publications/health/hs-niwts-tables-inpatient-and-day-case-waiting-q2-23-24.csv",
+# ---- Load internal sysdata.rda file with URLs ----
+load_all(".")
+
+# ---- Inpatient and Day Case waiting times ----
+query_url_inpatient <-
+  query_urls |>
+  filter(id == "ni_inpatient") |> # Each data release is cumulative
+  pull(query)
+
+ni_inpatient <- read_csv(query_url_inpatient,
   col_types = cols(
     .default = col_double(),
     `Quarter Ending` = col_character(),
@@ -13,9 +21,15 @@ ni_inpatient <- read_csv("https://www.health-ni.gov.uk/sites/default/files/publi
   )
 )
 
+# ---- Outpatient waiting times ----
+query_url_outpatient <-
+  query_urls |>
+  filter(id == "ni_outpatient") |> # Each data release is cumulative
+  pull(query)
+
 # Statistics by HSC Trust and Outpatients: https://www.health-ni.gov.uk/publications/northern-ireland-waiting-time-statistics-outpatient-waiting-times-march-2023
 # Previous Q: "https://www.health-ni.gov.uk/sites/default/files/publications/health/hs-niwts-tables-outpatients-q1-23-24.csv"
-ni_outpatient <- read_csv("https://www.health-ni.gov.uk/sites/default/files/publications/health/hs-niwts-tables-outpatients-q2-23-24.csv",
+ni_outpatient <- read_csv(query_url_outpatient,
   col_types = cols(
     .default = col_character(),
     `Quarter Ending` = col_character(),

data-raw/query-urls.R

@@ -88,13 +88,16 @@ query_urls <-
     "Bed availability", "ni_beds_pre22", "2013 - 2022", "OGLv3", "https://www.health-ni.gov.uk/sites/default/files/publications/health/hs-inpatient-hts-tables-21-22.csv", "https://www.health-ni.gov.uk/publications/hospital-statistics-inpatient-and-day-case-activity-202122",
     "Bed availability", "ni_beds_23", "2022 - 2023", "OGLv3", "https://www.health-ni.gov.uk/sites/default/files/publications/health/hs-inpatient-hts-tables-22-23.csv", "https://www.health-ni.gov.uk/publications/hospital-statistics-inpatient-and-day-case-activity-202223",
     "Provision of unpaid care (delayed discharge)", "ni_unpaid_care_21", "2021", "OGLv3", "https://www.nisra.gov.uk/system/files/statistics/census-2021-ms-d17.xlsx", "https://www.nisra.gov.uk/publications/census-2021-main-statistics-health-disability-and-unpaid-care-tables",
+    "Referral to Treatment", "ni_inpatient", "September 2023", "OGLv3", "https://www.health-ni.gov.uk/sites/default/files/publications/health/hs-niwts-tables-inpatient-and-day-case-waiting-q2-23-24.csv", "https://www.health-ni.gov.uk/articles/inpatient-waiting-times",
+    "Referral to Treatment", "ni_outpatient", "September 2023", "OGLv3", "https://www.health-ni.gov.uk/sites/default/files/publications/health/hs-niwts-tables-outpatients-q2-23-24.csv", "https://www.health-ni.gov.uk/articles/outpatient-waiting-times",
 
     # Scotland
     "Beds by Board of Treatment and Specialty", "scotland_beds", "2016 - 2021", "OGLv3", "https://www.opendata.nhs.scot/dataset/554b0e7a-ccac-4bb6-82db-1a8b306fcb36/resource/f272bb7d-5320-4491-84c1-614a2c064007/download/beds_by_nhs_board_of_treatment_and_specialty.csv", "https://www.opendata.nhs.scot/dataset/hospital-beds-information/resource/f272bb7d-5320-4491-84c1-614a2c064007",
     "Delayed Discharge Bed Days by Health Board", "scotland_delayed_discharge_hb_22", "2016 - 2022", "OGLv3", "https://www.opendata.nhs.scot/dataset/52591cba-fd71-48b2-bac3-e71ac108dfee/resource/fd354e4b-6211-48ba-8e4f-8356a5ed4215/download/2022-09_delayed-discharge-beddays-health-board.csv", "https://www.opendata.nhs.scot/dataset/delayed-discharges-in-nhsscotland/resource/fd354e4b-6211-48ba-8e4f-8356a5ed4215",
     "Delayed Discharge Bed Days by Health Board", "scotland_delayed_discharge_hb_23", "2016 - 2023", "OGLv3", "https://www.opendata.nhs.scot/dataset/52591cba-fd71-48b2-bac3-e71ac108dfee/resource/fd354e4b-6211-48ba-8e4f-8356a5ed4215/download/2023-11_delayed-discharge-beddays-health-board.csv", "https://www.opendata.nhs.scot/dataset/delayed-discharges-in-nhsscotland/resource/fd354e4b-6211-48ba-8e4f-8356a5ed4215",
     "Delayed Discharge Bed Days by Council Area", "scotland_delayed_discharge_ltla_22", "2016 - 2022", "OGLv3", "https://www.opendata.nhs.scot/dataset/52591cba-fd71-48b2-bac3-e71ac108dfee/resource/513d2d71-cf73-458e-8b44-4fa9bccbf50a/download/2023-03_delayed-discharge-beddays-council-area.csv", "https://www.opendata.nhs.scot/dataset/delayed-discharges-in-nhsscotland/resource/fd354e4b-6211-48ba-8e4f-8356a5ed4215",
     "Delayed Discharge Bed Days by Council Area", "scotland_delayed_discharge_ltla_23", "2016 - 2023", "OGLv3", "https://www.opendata.nhs.scot/dataset/52591cba-fd71-48b2-bac3-e71ac108dfee/resource/513d2d71-cf73-458e-8b44-4fa9bccbf50a/download/2023-11_delayed-discharge-beddays-council-area.csv", "https://www.opendata.nhs.scot/dataset/delayed-discharges-in-nhsscotland/resource/fd354e4b-6211-48ba-8e4f-8356a5ed4215",
+    "Referral to Treatment", "scotland_rtt", "2011 - ", "OGLv3", "https://www.opendata.nhs.scot/dataset/aa8b22e8-8a02-484d-a6c8-0a0154a6249d/resource/f2598c24-bf00-4171-b7ef-a469bbacbf6c/download/open_data_18_weeks_rtt.csv", "https://www.opendata.nhs.scot/dataset/18-weeks-referral-to-treatment", # File rewritten when updated monthly
 
     # Wales
     # Data from statswales.gov.wales is accessed via API

data-raw/scotland-delayed-discharge.R

@@ -16,9 +16,7 @@ scotland_delayed_discharge_hb <- read_csv(query_url)
 
 scotland_delayed_discharge_hb <-
   scotland_delayed_discharge_hb |>
-
   mutate(date = ym(MonthOfDelay)) |>
-
   select(
     hb_code = HBT,
     date,

data-raw/scotland-rtt.R

@@ -1,8 +1,16 @@
 library(tidyverse)
 library(lubridate)
 
-# Download monthly RTT data by Health Board from https://www.opendata.nhs.scot/dataset/18-weeks-referral-to-treatment
-raw <- read_csv("https://www.opendata.nhs.scot/dataset/aa8b22e8-8a02-484d-a6c8-0a0154a6249d/resource/f2598c24-bf00-4171-b7ef-a469bbacbf6c/download/open_data_18_weeks_rtt_march23.csv")
+# ---- Load internal sysdata.rda file with URLs ----
+pkgload::load_all(".")
+
+# ---- RTT ----
+query_url <-
+  query_urls |>
+  filter(id == "scotland_rtt") |> # Each data release is cumulative
+  pull(query)
+
+raw <- read_csv(query_url)
 
 scotland_rtt <- raw |>
   filter(HBT == "S92000003") |> # national waiting lists