From ad089cf00449b6591e3188b0bab242a3cf4c4b8e Mon Sep 17 00:00:00 2001
From: minhkhul <118945681+minhkhul@users.noreply.github.com>
Date: Tue, 27 Aug 2024 12:53:00 -0400
Subject: [PATCH 01/13] add hhs geo level (#2036)

---
 _delphi_utils_python/delphi_utils/geomap.py | 6 ++++--
 nssp/delphi_nssp/constants.py               | 1 +
 nssp/delphi_nssp/run.py                     | 8 ++++++++
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/_delphi_utils_python/delphi_utils/geomap.py b/_delphi_utils_python/delphi_utils/geomap.py
index be6df2d24..f6df4a919 100644
--- a/_delphi_utils_python/delphi_utils/geomap.py
+++ b/_delphi_utils_python/delphi_utils/geomap.py
@@ -443,7 +443,7 @@ def add_population_column(
         ---------
         data: pd.DataFrame
             The dataframe with a FIPS code column.
-        geocode_type: {"fips", "zip"}
+        geocode_type:
             The type of the geocode contained in geocode_col.
         geocode_col: str, default None
             The name of the column containing the geocodes. If None, uses the geocode_type
@@ -671,8 +671,10 @@ def aggregate_by_weighted_sum(
             to a from_geo, e.g. "wastewater collection site").
         to_geo: str
             The column name of the geocode to aggregate to.
-        sensor: str
+        sensor_col: str
             The column name of the sensor to aggregate.
+        time_col: str
+            The column name of the timestamp to aggregate over.
         population_column: str
             The column name of the population to weight the sensor by.
 
diff --git a/nssp/delphi_nssp/constants.py b/nssp/delphi_nssp/constants.py
index ddd2e74b8..9b98d2012 100644
--- a/nssp/delphi_nssp/constants.py
+++ b/nssp/delphi_nssp/constants.py
@@ -6,6 +6,7 @@
     "nation",
     "state",
     "county",
+    "hhs",
 ]
 
 SIGNALS_MAP = {
diff --git a/nssp/delphi_nssp/run.py b/nssp/delphi_nssp/run.py
index 7c5a3ffac..931cd7c71 100644
--- a/nssp/delphi_nssp/run.py
+++ b/nssp/delphi_nssp/run.py
@@ -111,6 +111,14 @@ def run_module(params):
                 df = geo_mapper.add_geocode(df, "fips", "msa", from_col="fips", new_col="geo_id")
                 df = geo_mapper.aggregate_by_weighted_sum(df, "geo_id", "val", "timestamp", "population")
                 df = df.rename(columns={"weighted_val": "val"})
+            elif geo == "hhs":
+                df = df[(df["county"] == "All") & (df["geography"] != "United States")]
+                df = df[["geography", "val", "timestamp"]]
+                df = geo_mapper.add_population_column(df, geocode_type="state_name", geocode_col="geography")
+                df = geo_mapper.add_geocode(df, "state_name", "state_code", from_col="state_name")
+                df = geo_mapper.add_geocode(df, "state_code", "hhs", from_col="state_code", new_col="geo_id")
+                df = geo_mapper.aggregate_by_weighted_sum(df, "geo_id", "val", "timestamp", "population")
+                df = df.rename(columns={"weighted_val": "val"})
             else:
                 df = df[df["county"] != "All"]
                 df["geo_id"] = df["fips"]

From d52dcd5a6b84083fb5cbca4164ebdd8429ac81d3 Mon Sep 17 00:00:00 2001
From: nmdefries <42820733+nmdefries@users.noreply.github.com>
Date: Wed, 28 Aug 2024 18:02:56 -0400
Subject: [PATCH 02/13] Add discussion of patching functionality to indicator
 manual (#2031)

* basic patching info

* custom_run

* add hhs to geo section

* de-emphasize R in contributing doc, and add links

* ref/issue date intro

* naming and force push standards

* secret and params links
---
 .github/CONTRIBUTING.md                 |  13 +--
 _template_python/INDICATOR_DEV_GUIDE.md | 104 ++++++++++++++++++------
 2 files changed, 88 insertions(+), 29 deletions(-)

diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
index 0dcfa85c2..66b0afc62 100644
--- a/.github/CONTRIBUTING.md
+++ b/.github/CONTRIBUTING.md
@@ -12,7 +12,9 @@ The production branch is configured to automatically deploy to our production en
 
 * everything else
 
-All other branches are development branches. We don't enforce a naming policy.
+All other branches are development branches. We don't enforce a naming policy, but it is recommended to prefix all branches you create with your name, username, or initials (e.g. `username/branch-name`).
+
+We don't forbid force-pushing, but please keep to a minimum and be careful of using when modifying a branch at the same time as others.
 
 ## Issues
 
@@ -29,7 +31,7 @@ So, how does one go about developing a pipeline for a new data source?
 ### tl;dr
 
 1. Create your new indicator branch from `main`.
-2. Build it using the appropriate template, following the guidelines in the included README.md and REVIEW.md files.
+2. Build it using the [indicator template](https://github.com/cmu-delphi/covidcast-indicators/tree/main/_template_python), following the guidelines in the included README.md, REVIEW.md, and INDICATOR_DEV_GUIDE.md files.
 3. Make some stuff!
 4. When your stuff works, push your development branch to remote, and open a PR against `main` for review.
 5. Once your PR has been merged, consult with a platform engineer for the remaining production setup needs. They will create a deployment workflow for your indicator including any necessary production parameters. Production secrets are encrypted in the Ansible vault. This workflow will be tested in staging by admins, who will consult you about any problems they encounter.
@@ -50,7 +52,7 @@ git checkout -b dev-my-feature-branch
 
 ### Creating your indicator
 
-Create a directory for your new indicator by making a copy of `_template_r` or `_template_python` depending on the programming language you intend to use. If using Python, add the name of the directory to the list found in `jobs > build > strategy > matrix > packages` in `.github/workflows/python-ci.yml`, which will enable automated checks for your indicator when you make PRs. The template copies of `README.md` and `REVIEW.md` include the minimum requirements for code structure, documentation, linting, testing, and method of configuration. Beyond that, we don't have any established restrictions on implementation; you can look at other existing indicators see some examples of code layout, organization, and general approach.
+Create a directory for your new indicator by making a copy of `_template_python`. (We also make a `_template_r` available, but R should be only used as a last resort, due to complications using it in production.) Add the name of the directory to the list found in `jobs > build > strategy > matrix > packages` in `.github/workflows/python-ci.yml`, which will enable automated checks for your indicator when you make PRs. The template copies of `README.md` and `REVIEW.md` include the minimum requirements for code structure, documentation, linting, testing, and method of configuration. Beyond that, we don't have any established restrictions on implementation; you can look at other existing indicators see some examples of code layout, organization, and general approach.
 
 * Consult your peers with questions! :handshake:
 
@@ -62,7 +64,7 @@ Once you have something that runs locally and passes tests you set up your remot
 git push -u origin dev-my-feature-branch
 ```
 
-You can then draft public API documentation for people who would fetch this
+You can then draft [public API documentation](https://cmu-delphi.github.io/delphi-epidata/) for people who would fetch this
 data from the API. Public API documentation is kept in the delphi-epidata
 repository, and there is a [template Markdown
 file](https://github.com/cmu-delphi/delphi-epidata/blob/main/docs/api/covidcast-signals/_source-template.md)
@@ -104,7 +106,8 @@ We use a branch-based git workflow coupled with [Jenkins](https://www.jenkins.io
   * Package - Tar and gzip the built environment.
   * Deploy - Trigger an Ansible playbook to place the built package onto the runtime host, place any necessary production configuration, and adjust the runtime envirnemnt (if necessary).
 
-There are several additional Jenkins-specific files that will need to be created for each indicator, as well as some configuration additions to the runtime host. It will be important to pair with a platform engineer to prepare the necessary production environment needs, test the workflow, validate on production, and ultimately sign off on a production release.
+There are several additional Jenkins-specific files that will need to be created for each indicator, as well as some configuration additions to the runtime host.
+It will be important to pair with a platform engineer to prepare the necessary production environment needs, test the workflow, validate on production, and ultimately sign off on a production release.
 
 ### Preparing container images of indicators
 
diff --git a/_template_python/INDICATOR_DEV_GUIDE.md b/_template_python/INDICATOR_DEV_GUIDE.md
index 2b8b6333c..f58a1ada9 100644
--- a/_template_python/INDICATOR_DEV_GUIDE.md
+++ b/_template_python/INDICATOR_DEV_GUIDE.md
@@ -50,6 +50,7 @@ This is the general extract-transform-load procedure used by all COVIDcast indic
 7. Deliver the CSV output files to the `receiving/` directory on the API server.
 
 Adding a new indicator typically means implementing steps 1-3. Step 4 is included via the function ` create_export_csv`. Steps 5 (the validator), 6 (the archive differ) and 7 (acquisition) are all handled by runners in production.
+
 ## Step 0: Keep revision history (important!)
 
 If the data provider doesn’t provide or it is unclear if they provide historical versions of the data, immediately set up a script (bash, Python, etc) to automatically (e.g. cron) download the data every day and save locally with versioning.
@@ -195,7 +196,14 @@ Generally, indicators have:
   Do other geo handling (e.g. finding and reporting DC as a state).
 * `constants.py`: Lists of geos to produce, signals to produce, dataset ids, data source URL, etc.
 
-Your code should be _extensively_ commented! Especially note sections where you took an unusual approach (make sure to say why and consider briefly discussing alternate approaches).
+Your code should be _extensively_ commented! Especially note sections where you took an unusual approach (make sure to say why and consider briefly discussing alternate approaches that were discarded or could be useful in the future).
+
+#### Development environment
+
+Make sure you have a functional environment with python 3.8.15+.
+For local runs, the makefile’s make install target will set up a local virtual environment with necessary packages.
+
+(If working in R (very much NOT recommended), local runs can be run without a virtual environment or using the [`renv` package](https://rstudio.github.io/renv/articles/renv.html), but production runs should be set up to use Docker.)
 
 #### Function stubs
 
@@ -235,12 +243,22 @@ def api_call(token: str):
 
 After that, generalize your code to be able to be run on all geos of interest, take settings from params.json, use constants for easy maintenance, with extensive documentation, etc.
 
-#### Development environment
+#### Testing
 
-Make sure you have a functional environment with python 3.8.15+.
-For local runs, the makefile’s make install target will set up a local virtual environment with necessary packages.
+As a general rule, it helps to decompose your functions into operations for which you can write unit tests.
+To run the tests, use `make test` in the top-level indicator directory.
 
-(If working in R (very much NOT recommended), local runs can be run without a virtual environment or using the [`renv` package](https://rstudio.github.io/renv/articles/renv.html), but production runs should be set up to use Docker.)
+Unit tests are required for all functions.
+Integration tests are highly desired, but may be difficult to set up depending on where the data is being fetched from.
+Mocking functions are useful in this case.
+
+#### Dealing with dates
+
+We keep track of two different date fields for each dataset. The first field is called "reference value" (field name `time_value`) and tracks the date that a value is reported _for_, that is, when the event happened. The second field is called "issue date" or "version" (field name `issue`) and tracks when a value was recorded, not when it happened.
+
+For example, flu test positivity of 80% for a reference date of Jan 1 and an issue date of Jan 5 means that _on_ Jan 1, the test positivity rate was 80%. But we only received and recorded the value on Jan 5, 4 days later (AKA a lag of 4 days).
+
+It's important to track issue date because many data sources are revised over time, and reported values can change substantially between issues.
 
 #### Dealing with data-types
 
@@ -255,14 +273,16 @@ E.g. which geo values are allowed, should every valid date be present in some wa
 
 In an ideal case, the data exists at one of our [already covered geos](https://cmu-delphi.github.io/delphi-epidata/api/covidcast_geography.html):
 
-* State: state_code (string, leftpadded to 2 digits with 0) or state_id (string)
+* Zip code
 * FIPS (state+county codes, string leftpadded to 5 digits with 0)
-* ZIP
 * MSA (metro statistical area, int)
 * HRR (hospital referral region, int)
+* State: state_code (string, leftpadded to 2 digits with 0) or state_id (string)
+* HHS ([Department of Health and Human Services-defined regions](https://www.hhs.gov/about/agencies/iea/regional-offices/index.html))
+* Nation
 
-If you want to map from one of these to another, the [`delphi_utils.geomapper`](https://github.com/cmu-delphi/covidcast-indicators/blob/6912077acba97e835aff7d0cd3d64309a1a9241d/_delphi_utils_python/delphi_utils/geomap.py) utility covers most cases.
-A brief example of aggregating from states to hhs regions via their population:
+If you want to map from one of these to a higher level, the [`delphi_utils.geomapper`](https://github.com/cmu-delphi/covidcast-indicators/blob/6912077acba97e835aff7d0cd3d64309a1a9241d/_delphi_utils_python/delphi_utils/geomap.py) utility covers most cases.
+Here's a brief example of aggregating from states to hhs regions via their population:
 
 ```{python}
 from delphi_utils.geomap import GeoMapper
@@ -274,19 +294,6 @@ hhs_version = geo_mapper.replace_geocode(df, "state_code","hhs", new_col = "geo_
 
 This example is taken from [`hhs_hosp`](https://github.com/cmu-delphi/covidcast-indicators/blob/main/hhs_hosp/delphi_hhs/run.py); more documentation can be found in the `geomapper` class definition.
 
-#### Implement a Missing Value code system
-
-The column is described [here](https://cmu-delphi.github.io/delphi-epidata/api/missing_codes.html).
-
-#### Local testing
-
-As a general rule, it helps to decompose your functions into operations for which you can write unit tests.
-To run the tests, use `make test` in the top-level indicator directory.
-
-Unit tests are required for all functions.
-Integration tests are highly desired, but may be difficult to set up depending on where the data is being fetched from.
-Mocking functions are useful in this case.
-
 #### Naming
 
 Indicator and signal names need to be approved by [@RoniRos](https://www.github.com/RoniRos).
@@ -324,6 +331,52 @@ Using this tag dictionary, we can interpret the following signals as
 * `confirmed_admissions_influenza_1d_prop` = raw (unsmoothed) daily ("1d") confirmed influenza hospital admissions ("confirmed_admissions_influenza") per 100,000 population ("prop").
 * `confirmed_admissions_influenza_1d_prop_7dav` = the same as above, but smoothed with a 7-day moving average ("7dav").
 
+#### Implement a Missing Value code system
+
+The column is described [here](https://cmu-delphi.github.io/delphi-epidata/api/missing_codes.html).
+
+#### Implement a patching method
+
+After normal data reporting is restored following an outage, we would like to be able to easily reconstruct the version history of the data. To do so, implement a `patch` method that runs an indicator's main `run_module` for every issue date in a range. An [example patch module](https://github.com/cmu-delphi/covidcast-indicators/blob/b784f30/google_symptoms/delphi_google_symptoms/patch.py).
+
+An outage can be external to Delphi, e.g. the data provider was unable to provide new data on the historically-expected schedule, or internal, e.g. Delphi code had a bug that caused our pipeline to fail. The goal of the patch feature is to recreate every missing issue _as if we had ingested it on the correct day_.
+
+The patch feature should be easy to use. The only manual parts should be modifying `params.json`, and running the patch module and acquisition. Any setup that needs to be done (e.g. cache creation, dir creation) should be done automatically as part of the patch function.
+
+All patch modules should expect settings from `params.json` of the form
+
+```
+{
+  "common": {
+    ...
+    "custom_run": true
+  },
+  "validation": {
+    ...
+  },
+  "patch": {
+    "patch_dir": "<path to dir>/<patch dir name>",
+    "start_issue": "2024-04-20",
+    "end_issue": "2024-04-21"
+  }
+}
+```
+
+The `custom_run` parameter should [default to false](https://github.com/cmu-delphi/covidcast-indicators/blob/d435bf0f0d5880ddf8905ea60f242976e6702342/nssp/delphi_nssp/run.py#L73), and [warn](https://github.com/cmu-delphi/covidcast-indicators/blob/d435bf0f0d5880ddf8905ea60f242976e6702342/nssp/delphi_nssp/run.py#L75-L83) if parameters and arguments disagree.
+
+Patching should generate data for that range of issue dates, and store them in batch issue format:
+`<patch_dir as provided in the params>/issue_<issue date>/<indicator name as stored in our DB>/xxx.csv`.
+
+Acquisition in `delphi-epidata` includes [code that allow files in this issue-specific structure](https://github.com/cmu-delphi/delphi-epidata/blob/694d89ad763fa85bd644e1f64552c9bc85f688ef/src/acquisition/covidcast/csv_to_database.py#L43C32-L43C61) to be added to the database. This output format is designed to match the `issue`-type acquisition format. The issue-specific mode is triggered with the flag `specific_issue_date`. [A Cronicle job](https://cronicle-prod-01.delphi.cmu.edu/#Schedule?sub=edit_event&id=elh59ynwobf) has already been set up to call acquisition using the flag; please use it to load patches into the database.
+
+Sometimes source data is already versioned, and to reconstruct an issue we simply need to filter the source data to include only values that would have been available on that issue day. If we receive data drops directly, we can filter by the file creation date instead.
+
+However, it is not always possible to reconstruct issues; many datasets aren't versioned by the provider. If a source has no revisions (for example, `google-symptoms`), then we can guess which dates of data would have been available that issue day based on the normal lag of the source. For example, `google-symptoms` normally has a lag of 4 days, i.e. "today" the most recent data we see in the source data is from 4 days ago. So to reconstruct data for issue 2024-01-10, we just need to report data with a `time_value` (reference date) from 2024-01-06 and earlier. (How much earlier depends on the behavior we normally expect from the indicator code; if we normally report 2 weeks of data, filter to 2024-01-06 - 14 days through 2024-01-06.)
+
+Some datasets, such as those on healthdata.gov, provide metadata indicating when certain rows were updated.
+
+In other cases (such as datasetes that both have revisions _and_ don't track revisions), please discuss with the indicator stakeholder and consider [what you know about how the data works](#step-1-exploratory-analysis).
+
 ### Statistical review
 
 The data produced by the new indicator needs to be sanity-checked.
@@ -419,14 +472,17 @@ Refer to [this guide](https://docs.google.com/document/d/1Bbuvtoxowt7x2_8USx_JY-
 
 * Add module name to the `build` job in `.github/workflows/python-ci.yml`.
   This allows github actions to run on this indicator code, which includes unit tests and linting.
-* Add top-level directory name to `indicator_list` in `Jenkinsfile`.
+* Add module name to the ["Copy version to indicator directory" step](https://github.com/cmu-delphi/covidcast-indicators/blob/f01185767a9847d8082baf4f1e17be50a39047c2/.github/workflows/create-release.yml#L64) in `.github/workflows/create-release.yml`.
+* Add top-level directory name to [`indicator_list` in `Jenkinsfile`](https://github.com/cmu-delphi/covidcast-indicators/blob/f01185767a9847d8082baf4f1e17be50a39047c2/Jenkinsfile#L13).
   This allows your code to be automatically deployed to staging after your branch is merged to main, and deployed to prod after `covidcast-indicators` is released.
 * Create `ansible/templates/{top_level_directory_name}-params-prod.json.j2` based on your `params.json.template` with some adjustment:
    * "export_dir": "/common/covidcast/receiving/{data-source-name}"
    * "log_filename": "/var/log/indicators/{top_level_directory_name}.log"
+* Define any sensitive variables as "secrets" in the [Ansible `vars.yaml`](https://github.com/cmu-delphi/covidcast-indicators/blob/main/ansible/vars.yaml) and [vault](https://github.com/cmu-delphi/covidcast-indicators/blob/main/ansible/vault.yaml).
+Refer to [this guide](https://docs.google.com/document/d/1Bbuvtoxowt7x2_8USx_JY-yTo-Av3oAFlhyG-vXGG-c/edit#heading=h.8kkoy8sx3t7f) for more vault info.
+* Add configs for Sir Complains-a-Lot ("sirCAL") alerting in sirCAL's [local](https://github.com/cmu-delphi/covidcast-indicators/blob/main/sir_complainsalot/params.json.template) and [Ansible](https://github.com/cmu-delphi/covidcast-indicators/blob/main/ansible/templates/sir_complainsalot-params-prod.json.j2) params templates.
 
 Pay attention to the receiving/export directory, as well as how you can store credentials in vault.
-Refer to [this guide](https://docs.google.com/document/d/1Bbuvtoxowt7x2_8USx_JY-yTo-Av3oAFlhyG-vXGG-c/edit#heading=h.8kkoy8sx3t7f) for more vault info.
 
 ### Staging
 

From 86399ad65e774c1ae79665a80f923e69a1dec575 Mon Sep 17 00:00:00 2001
From: george <george.haff@gmail.com>
Date: Mon, 9 Sep 2024 10:15:37 -0400
Subject: [PATCH 03/13] Clean up delphi_utils (take 2) (#2014)

* add pyproject.toml

* made changes based on suggestion

* update ci config

* separate new line

Co-authored-by: Dmitry Shemetov <dshemetov@ucdavis.edu>

* fix: add explicit boto and moto dependencies to indicators

* trying to get package data to show

* installing dev package before testing in ci

* adding dependency

* move comment to right package

* Update _delphi_utils_python/pyproject.toml

Co-authored-by: Dmitry Shemetov <dshemetov@ucdavis.edu>

* more suggestion

* adding pyproject.toml to template

* Update changehc/Makefile

* Update claims_hosp/Makefile

* Update doctor_visits/Makefile

* Update google_symptoms/Makefile

* Update hhs_hosp/Makefile

* Update nchs_mortality/Makefile

* Update nssp/Makefile

* Update nwss_wastewater/Makefile

* Update quidel_covidtest/Makefile

* Update sir_complainsalot/Makefile

* fix: dependence bugs and cleanup
* pandas[excel]<2 doesn't work, go back to xlrd and comment
* add mock to changehc
* add requests
* remove setup.py from template

* build: fix Jenkins build script

---------

Co-authored-by: Amaris Sim <aysim319@gmail.com>
Co-authored-by: aysim319 <aysim319@users.noreply.github.com>
Co-authored-by: Dmitry Shemetov <dshemetov@ucdavis.edu>
---
 .github/workflows/python-ci.yml       |  2 +-
 _delphi_utils_python/.bumpversion.cfg |  2 +-
 _delphi_utils_python/DEVELOP.md       |  2 +-
 _delphi_utils_python/Makefile         |  9 +++--
 _delphi_utils_python/pyproject.toml   | 57 +++++++++++++++++++++++++++
 _delphi_utils_python/setup.py         | 48 ----------------------
 _template_python/pyproject.toml       | 42 ++++++++++++++++++++
 _template_python/setup.py             | 30 --------------
 changehc/setup.py                     |  1 +
 jenkins/build-indicator.sh            |  4 +-
 nchs_mortality/setup.py               |  2 +
 quidel_covidtest/setup.py             |  3 +-
 12 files changed, 114 insertions(+), 88 deletions(-)
 create mode 100644 _delphi_utils_python/pyproject.toml
 delete mode 100644 _delphi_utils_python/setup.py
 create mode 100644 _template_python/pyproject.toml
 delete mode 100644 _template_python/setup.py

diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
index 3e1ee9689..6e23c61d2 100644
--- a/.github/workflows/python-ci.yml
+++ b/.github/workflows/python-ci.yml
@@ -51,7 +51,7 @@ jobs:
         with:
           python-version: 3.8
           cache: "pip"
-          cache-dependency-path: "setup.py"
+          cache-dependency-path: "pyproject.toml"
       - name: Install testing dependencies
         run: |
           python -m pip install --upgrade pip
diff --git a/_delphi_utils_python/.bumpversion.cfg b/_delphi_utils_python/.bumpversion.cfg
index 722a91e30..af22c56eb 100644
--- a/_delphi_utils_python/.bumpversion.cfg
+++ b/_delphi_utils_python/.bumpversion.cfg
@@ -4,6 +4,6 @@ commit = True
 message = chore: bump delphi_utils to {new_version}
 tag = False
 
-[bumpversion:file:setup.py]
+[bumpversion:file:pyproject.toml]
 
 [bumpversion:file:delphi_utils/__init__.py]
diff --git a/_delphi_utils_python/DEVELOP.md b/_delphi_utils_python/DEVELOP.md
index 2407e29a8..53ffde93f 100644
--- a/_delphi_utils_python/DEVELOP.md
+++ b/_delphi_utils_python/DEVELOP.md
@@ -9,7 +9,7 @@ To install the module in your default version of Python, run the
 following from this directory:
 
 ```
-pip install .
+pip install -e '.[dev]'
 ```
 
 As described in each of the indicator code directories, you will want to install
diff --git a/_delphi_utils_python/Makefile b/_delphi_utils_python/Makefile
index 79d7f7943..60f7d2af0 100644
--- a/_delphi_utils_python/Makefile
+++ b/_delphi_utils_python/Makefile
@@ -6,12 +6,12 @@ venv:
 install: venv
 	. env/bin/activate; \
 	pip install wheel ; \
-	pip install -e .
+	pip install -e '.[dev]'
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
-	pip install .
+	pip install 'build[virtualenv]' pylint pytest pydocstyle wheel twine ; \
+	pip install '.[dev]'
 
 lint:
 	. env/bin/activate; pylint delphi_utils --rcfile=../pyproject.toml
@@ -30,4 +30,5 @@ clean:
 
 release: lint test
 	. env/bin/activate ; \
-	python setup.py sdist bdist_wheel
+	pip install 'build[virtualenv]' ; \
+	python -m build --sdist --wheel
\ No newline at end of file
diff --git a/_delphi_utils_python/pyproject.toml b/_delphi_utils_python/pyproject.toml
new file mode 100644
index 000000000..d6480baf0
--- /dev/null
+++ b/_delphi_utils_python/pyproject.toml
@@ -0,0 +1,57 @@
+[build-system]
+requires = ["setuptools", "setuptools-scm>=8.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "delphi-utils"
+version = "0.3.24"
+description = "Shared Utility Functions for Indicators"
+readme = "README.md"
+requires-python = "== 3.8.*"
+license = { text = "MIT License" }
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3.8",
+    "License :: MIT",
+]
+dependencies = [
+    "boto3",
+    "covidcast",
+    "cvxpy",
+    "epiweeks",
+    "gitpython",
+    "importlib_resources>=1.3",
+    "numpy",
+    "pandas>=1.1.0",
+    "requests",
+    "slackclient",
+    "scs<3.2.6",                # TODO: remove this ; it is a cvxpy dependency, and the excluded version appears to break our jenkins build. see: https://github.com/cvxgrp/scs/issues/283
+    "structlog",
+    "xlrd",                     # needed by Pandas to read Excel files
+]
+
+[project.urls]
+Homepage = "https://github.com/cmu-delphi/covidcast-indicators"
+
+[project.optional-dependencies]
+dev = [
+    "darker[isort]~=2.1.1",
+    "pylint==2.8.3",
+    "pytest",
+    "pydocstyle",
+    "pytest-cov",
+    "mock",
+    "moto~=4.2.14",
+    "requests-mock",
+    "freezegun",
+]
+flash = ["scipy"]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["delphi_utils"]
+namespaces = true
+
+[tool.setuptools.package-data]
+"delphi_utils.data" = ["20*/*.csv"]
diff --git a/_delphi_utils_python/setup.py b/_delphi_utils_python/setup.py
deleted file mode 100644
index 3dee89b53..000000000
--- a/_delphi_utils_python/setup.py
+++ /dev/null
@@ -1,48 +0,0 @@
-from setuptools import setup
-from setuptools import find_packages
-
-with open("README.md", "r") as f:
-    long_description = f.read()
-
-required = [
-    "boto3",
-    "covidcast",
-    "cvxpy",
-    "scs<3.2.6", # TODO: remove this ; it is a cvxpy dependency, and the excluded version appears to break our jenkins build. see: https://github.com/cvxgrp/scs/issues/283
-    "darker[isort]~=2.1.1",
-    "epiweeks",
-    "freezegun",
-    "gitpython",
-    "importlib_resources>=1.3",
-    "mock",
-    "moto~=4.2.14",
-    "numpy",
-    "pandas>=1.1.0",
-    "pydocstyle",
-    "pylint==2.8.3",
-    "pytest-cov",
-    "pytest",
-    "requests-mock",
-    "slackclient",
-    "structlog",
-    "xlrd"
-]
-
-setup(
-    name="delphi_utils",
-    version="0.3.24",
-    description="Shared Utility Functions for Indicators",
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    author="",
-    author_email="",
-    url="https://github.com/cmu-delphi/",
-    install_requires=required,
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "Intended Audience :: Developers",
-        "Programming Language :: Python :: 3.8",
-    ],
-    packages=find_packages(),
-    package_data={'': ['data/20*/*.csv']}
-)
diff --git a/_template_python/pyproject.toml b/_template_python/pyproject.toml
new file mode 100644
index 000000000..c29d9650e
--- /dev/null
+++ b/_template_python/pyproject.toml
@@ -0,0 +1,42 @@
+[build-system]
+requires = ["setuptools", "setuptools-scm>=8.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "delphi_NAME"
+version = "0.1.0"
+description = "SHORT DESCRIPTION"
+readme = "README.md"
+requires-python = "== 3.8.*"
+license = { text = "MIT License" }
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3.8",
+    "License :: MIT",
+]
+dependencies = [
+    "delphi-utils",
+    "numpy",
+    "pandas>=1.1.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/cmu-delphi/covidcast-indicators"
+
+[project.optional-dependencies]
+dev = [
+    "darker[isort]~=2.1.1",
+    "pylint==2.8.3",
+    "pytest",
+    "pydocstyle",
+    "pytest-cov",
+    "mock",
+    "moto~=4.2.14",
+    "requests-mock",
+    "freezegun",
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+namespaces = true
diff --git a/_template_python/setup.py b/_template_python/setup.py
deleted file mode 100644
index d7bc44078..000000000
--- a/_template_python/setup.py
+++ /dev/null
@@ -1,30 +0,0 @@
-from setuptools import setup
-from setuptools import find_packages
-
-required = [
-    "covidcast",
-    "darker[isort]~=2.1.1",
-    "delphi-utils",
-    "numpy",
-    "pandas",
-    "pydocstyle",
-    "pylint==2.8.3",
-    "pytest-cov",
-    "pytest",
-]
-
-setup(
-    name="delphi_NAME",
-    version="0.1.0",
-    description="SHORT DESCRIPTION",
-    author="",
-    author_email="",
-    url="https://github.com/cmu-delphi/covidcast-indicators",
-    install_requires=required,
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "Intended Audience :: Developers",
-        "Programming Language :: Python :: 3.8",
-    ],
-    packages=find_packages(),
-)
diff --git a/changehc/setup.py b/changehc/setup.py
index d95beb771..f386e9613 100644
--- a/changehc/setup.py
+++ b/changehc/setup.py
@@ -6,6 +6,7 @@
     "covidcast",
     "darker[isort]~=2.1.1",
     "delphi-utils",
+    "mock",
     "moto~=4.2.14",
     "numpy",
     "pandas",
diff --git a/jenkins/build-indicator.sh b/jenkins/build-indicator.sh
index 3d2315741..84006767c 100755
--- a/jenkins/build-indicator.sh
+++ b/jenkins/build-indicator.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Jenkins build 
+# Jenkins build
 #
 
 set -eo pipefail
@@ -17,4 +17,4 @@ source env/bin/activate
 pip install pip==23.0.1 --retries 10 --timeout 20
 pip install numpy --retries 10 --timeout 20
 pip install ../_delphi_utils_python/. --retries 10 --timeout 20
-[ ! -f setup.py ] || pip install . --retries 10 --timeout 20
+[ ! -f pyproject.toml ] || pip install . --retries 10 --timeout 20
diff --git a/nchs_mortality/setup.py b/nchs_mortality/setup.py
index 3fe354ba4..142db6f35 100644
--- a/nchs_mortality/setup.py
+++ b/nchs_mortality/setup.py
@@ -2,10 +2,12 @@
 from setuptools import find_packages
 
 required = [
+    "boto3",
     "darker[isort]~=2.1.1",
     "delphi-utils",
     "epiweeks",
     "freezegun",
+    "moto~=4.2.14",
     "numpy",
     "pandas",
     "pydocstyle",
diff --git a/quidel_covidtest/setup.py b/quidel_covidtest/setup.py
index c2791930f..82c80832a 100644
--- a/quidel_covidtest/setup.py
+++ b/quidel_covidtest/setup.py
@@ -2,6 +2,7 @@
 from setuptools import find_packages
 
 required = [
+    "boto3",
     "covidcast",
     "darker[isort]~=2.1.1",
     "delphi-utils",
@@ -14,7 +15,7 @@
     "pylint==2.8.3",
     "pytest-cov",
     "pytest",
-    "xlrd==1.2.0",
+    "xlrd==1.2.0", # needed by Pandas to read Excel files
 ]
 
 setup(

From 6f841bb10a203ffa50fa7adf122085f738c22ad4 Mon Sep 17 00:00:00 2001
From: Dmitry Shemetov <dshemetov@ucdavis.edu>
Date: Wed, 11 Sep 2024 13:54:39 -0700
Subject: [PATCH 04/13] fix: build-indicators.sh looks for setup.py and
 pyproject.toml (#2048)

* fix: build-indicators.sh looks for setup.py again

Reverting a premature change.

* fix: readability and future-proof

* fix: more readability
---
 jenkins/build-indicator.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/jenkins/build-indicator.sh b/jenkins/build-indicator.sh
index 84006767c..88f79424a 100755
--- a/jenkins/build-indicator.sh
+++ b/jenkins/build-indicator.sh
@@ -17,4 +17,6 @@ source env/bin/activate
 pip install pip==23.0.1 --retries 10 --timeout 20
 pip install numpy --retries 10 --timeout 20
 pip install ../_delphi_utils_python/. --retries 10 --timeout 20
-[ ! -f pyproject.toml ] || pip install . --retries 10 --timeout 20
+if [ -f setup.py ] || [ -f pyproject.toml ]; then
+    pip install . --retries 10 --timeout 20
+fi

From 0da7b9aaa8715bab81d7023d48faa787f0c82851 Mon Sep 17 00:00:00 2001
From: minhkhul <118945681+minhkhul@users.noreply.github.com>
Date: Thu, 12 Sep 2024 14:51:19 -0400
Subject: [PATCH 05/13] upload-artifact@v2 -> v4 (#2051)

---
 .github/workflows/publish-release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml
index a9f1ccae9..e7cd8548b 100644
--- a/.github/workflows/publish-release.yml
+++ b/.github/workflows/publish-release.yml
@@ -86,7 +86,7 @@ jobs:
       - name: Release
         run: |
           make release
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: delphi_utils
           path: _delphi_utils_python/dist/*.tar.gz

From 2c597d2e6338caefdceebbecc47520b4a33fb6c7 Mon Sep 17 00:00:00 2001
From: Delphi Deploy Bot <delphibot@example.com>
Date: Thu, 12 Sep 2024 20:01:58 +0000
Subject: [PATCH 06/13] chore: bump delphi_utils to 0.3.25

---
 _delphi_utils_python/.bumpversion.cfg         | 2 +-
 _delphi_utils_python/delphi_utils/__init__.py | 2 +-
 _delphi_utils_python/pyproject.toml           | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/_delphi_utils_python/.bumpversion.cfg b/_delphi_utils_python/.bumpversion.cfg
index af22c56eb..08040f0ec 100644
--- a/_delphi_utils_python/.bumpversion.cfg
+++ b/_delphi_utils_python/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.24
+current_version = 0.3.25
 commit = True
 message = chore: bump delphi_utils to {new_version}
 tag = False
diff --git a/_delphi_utils_python/delphi_utils/__init__.py b/_delphi_utils_python/delphi_utils/__init__.py
index 7ff828440..7a418551d 100644
--- a/_delphi_utils_python/delphi_utils/__init__.py
+++ b/_delphi_utils_python/delphi_utils/__init__.py
@@ -15,4 +15,4 @@
 from .nancodes import Nans
 from .weekday import Weekday
 
-__version__ = "0.3.24"
+__version__ = "0.3.25"
diff --git a/_delphi_utils_python/pyproject.toml b/_delphi_utils_python/pyproject.toml
index d6480baf0..f7f14e8c2 100644
--- a/_delphi_utils_python/pyproject.toml
+++ b/_delphi_utils_python/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "delphi-utils"
-version = "0.3.24"
+version = "0.3.25"
 description = "Shared Utility Functions for Indicators"
 readme = "README.md"
 requires-python = "== 3.8.*"

From db9ea67abb4f54381d3f6f378973f391c4e56d26 Mon Sep 17 00:00:00 2001
From: Delphi Deploy Bot <delphibot@example.com>
Date: Thu, 12 Sep 2024 20:01:59 +0000
Subject: [PATCH 07/13] chore: bump covidcast-indicators to 0.3.56

---
 .bumpversion.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index de364861c..547bad12d 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.55
+current_version = 0.3.56
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

From 970307371eb79028922b0b02ac77da77dd1f192a Mon Sep 17 00:00:00 2001
From: melange396 <melange396@users.noreply.github.com>
Date: Thu, 12 Sep 2024 20:01:59 +0000
Subject: [PATCH 08/13] [create-pull-request] automated change

---
 changehc/version.cfg          | 2 +-
 claims_hosp/version.cfg       | 2 +-
 doctor_visits/version.cfg     | 2 +-
 google_symptoms/version.cfg   | 2 +-
 hhs_hosp/version.cfg          | 2 +-
 nchs_mortality/version.cfg    | 2 +-
 quidel_covidtest/version.cfg  | 2 +-
 sir_complainsalot/version.cfg | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/changehc/version.cfg b/changehc/version.cfg
index f5c28d2cd..ed4b085f1 100644
--- a/changehc/version.cfg
+++ b/changehc/version.cfg
@@ -1 +1 @@
-current_version = 0.3.55
+current_version = 0.3.56
diff --git a/claims_hosp/version.cfg b/claims_hosp/version.cfg
index f5c28d2cd..ed4b085f1 100644
--- a/claims_hosp/version.cfg
+++ b/claims_hosp/version.cfg
@@ -1 +1 @@
-current_version = 0.3.55
+current_version = 0.3.56
diff --git a/doctor_visits/version.cfg b/doctor_visits/version.cfg
index f5c28d2cd..ed4b085f1 100644
--- a/doctor_visits/version.cfg
+++ b/doctor_visits/version.cfg
@@ -1 +1 @@
-current_version = 0.3.55
+current_version = 0.3.56
diff --git a/google_symptoms/version.cfg b/google_symptoms/version.cfg
index f5c28d2cd..ed4b085f1 100644
--- a/google_symptoms/version.cfg
+++ b/google_symptoms/version.cfg
@@ -1 +1 @@
-current_version = 0.3.55
+current_version = 0.3.56
diff --git a/hhs_hosp/version.cfg b/hhs_hosp/version.cfg
index f5c28d2cd..ed4b085f1 100644
--- a/hhs_hosp/version.cfg
+++ b/hhs_hosp/version.cfg
@@ -1 +1 @@
-current_version = 0.3.55
+current_version = 0.3.56
diff --git a/nchs_mortality/version.cfg b/nchs_mortality/version.cfg
index f5c28d2cd..ed4b085f1 100644
--- a/nchs_mortality/version.cfg
+++ b/nchs_mortality/version.cfg
@@ -1 +1 @@
-current_version = 0.3.55
+current_version = 0.3.56
diff --git a/quidel_covidtest/version.cfg b/quidel_covidtest/version.cfg
index f5c28d2cd..ed4b085f1 100644
--- a/quidel_covidtest/version.cfg
+++ b/quidel_covidtest/version.cfg
@@ -1 +1 @@
-current_version = 0.3.55
+current_version = 0.3.56
diff --git a/sir_complainsalot/version.cfg b/sir_complainsalot/version.cfg
index f5c28d2cd..ed4b085f1 100644
--- a/sir_complainsalot/version.cfg
+++ b/sir_complainsalot/version.cfg
@@ -1 +1 @@
-current_version = 0.3.55
+current_version = 0.3.56

From 5c72cbd9985579382c88be271c2c68699bcd1be8 Mon Sep 17 00:00:00 2001
From: minhkhul <118945681+minhkhul@users.noreply.github.com>
Date: Thu, 12 Sep 2024 17:40:54 -0400
Subject: [PATCH 09/13] Correct pypi classifier string (#2055)

---
 _delphi_utils_python/pyproject.toml | 2 +-
 _template_python/pyproject.toml     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/_delphi_utils_python/pyproject.toml b/_delphi_utils_python/pyproject.toml
index f7f14e8c2..c47590a29 100644
--- a/_delphi_utils_python/pyproject.toml
+++ b/_delphi_utils_python/pyproject.toml
@@ -13,7 +13,7 @@ classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Developers",
     "Programming Language :: Python :: 3.8",
-    "License :: MIT",
+    "License :: OSI Approved :: MIT License",
 ]
 dependencies = [
     "boto3",
diff --git a/_template_python/pyproject.toml b/_template_python/pyproject.toml
index c29d9650e..2f84aaa00 100644
--- a/_template_python/pyproject.toml
+++ b/_template_python/pyproject.toml
@@ -13,7 +13,7 @@ classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Developers",
     "Programming Language :: Python :: 3.8",
-    "License :: MIT",
+    "License :: OSI Approved :: MIT License",
 ]
 dependencies = [
     "delphi-utils",

From 12a97123506ece1f383109b24fb6558655364928 Mon Sep 17 00:00:00 2001
From: aysim319 <aysim319@users.noreply.github.com>
Date: Wed, 18 Sep 2024 17:41:41 -0400
Subject: [PATCH 10/13] Fix unstructured logging variable formatting (#2029)

Co-authored-by: george <george.haff@gmail.com>
Co-authored-by: minhkhul <118945681+minhkhul@users.noreply.github.com>
---
 _delphi_utils_python/README.md                | 11 +++++-
 _delphi_utils_python/delphi_utils/export.py   |  6 ++--
 .../delphi_utils/flash_eval/eval_day.py       |  3 +-
 _delphi_utils_python/delphi_utils/logger.py   | 10 +++---
 _delphi_utils_python/delphi_utils/runner.py   | 19 +++++++---
 _delphi_utils_python/tests/test_export.py     | 10 +++---
 changehc/delphi_changehc/run.py               | 34 +++++++++---------
 changehc/delphi_changehc/sensor.py            | 14 ++++----
 changehc/delphi_changehc/update_sensor.py     | 19 +++++-----
 changehc/tests/test_update_sensor.py          |  3 +-
 .../download_claims_ftp_files.py              |  2 +-
 claims_hosp/delphi_claims_hosp/indicator.py   | 11 +++---
 .../delphi_claims_hosp/modify_claims_drops.py |  2 +-
 claims_hosp/delphi_claims_hosp/run.py         | 12 +++----
 .../delphi_claims_hosp/update_indicator.py    | 26 +++++++-------
 claims_hosp/tests/test_update_indicator.py    | 21 ++++++-----
 .../download_claims_ftp_files.py              |  2 +-
 .../modify_claims_drops.py                    |  2 +-
 doctor_visits/delphi_doctor_visits/patch.py   | 11 +++---
 doctor_visits/delphi_doctor_visits/run.py     | 35 ++++++++++---------
 doctor_visits/delphi_doctor_visits/sensor.py  |  2 +-
 .../delphi_doctor_visits/update_sensor.py     |  6 ++--
 .../delphi_google_symptoms/date_utils.py      |  2 +-
 .../delphi_google_symptoms/patch.py           | 11 +++---
 google_symptoms/delphi_google_symptoms/run.py |  9 ++---
 nssp/delphi_nssp/run.py                       |  2 +-
 nwss_wastewater/delphi_nwss/run.py            |  2 +-
 .../delphi_quidel_covidtest/pull.py           |  6 ++--
 .../delphi_quidel_covidtest/run.py            |  2 +-
 29 files changed, 163 insertions(+), 132 deletions(-)

diff --git a/_delphi_utils_python/README.md b/_delphi_utils_python/README.md
index 0ac6350bf..e1b091cae 100644
--- a/_delphi_utils_python/README.md
+++ b/_delphi_utils_python/README.md
@@ -22,13 +22,22 @@ Source code can be found here:
 
 ## Logger Usage
 
+To make our structured logging as useful as it can be, particularly within the context of how we use logs in Elastic, the `event` argument (typically the first unnamed arg) should be a static string (to make filtering easier), and each dynamic/varying value should be specified in an individual meaningfully- and consistently-named argument to the logger call (for use in filtering, thresholding, grouping, visualization, etc).
+
+### Commonly used argument names:
+- data_source
+- geo_type
+- signal
+- issue_date
+- filename
+
 Single-thread usage.
 
 ```py
 from delphi_utils.logger import get_structured_logger
 
 logger = get_structured_logger('my_logger')
-logger.info('Hello, world!')
+logger.info('Hello', name='World')
 ```
 
 Multi-thread usage.
diff --git a/_delphi_utils_python/delphi_utils/export.py b/_delphi_utils_python/delphi_utils/export.py
index 37f8faf98..8ac5de48e 100644
--- a/_delphi_utils_python/delphi_utils/export.py
+++ b/_delphi_utils_python/delphi_utils/export.py
@@ -22,8 +22,10 @@ def filter_contradicting_missing_codes(df, sensor, metric, date, logger=None):
     for mask in masks:
         if not logger is None and df.loc[mask].size > 0:
             logger.info(
-                "Filtering contradictory missing code in " +
-                "{0}_{1}_{2}.".format(sensor, metric, date.strftime(format="%Y-%m-%d"))
+                "Filtering contradictory missing code",
+                sensor=sensor,
+                metric=metric,
+                date=date.strftime(format="%Y-%m-%d"),
             )
             df = df.loc[~mask]
         elif logger is None and df.loc[mask].size > 0:
diff --git a/_delphi_utils_python/delphi_utils/flash_eval/eval_day.py b/_delphi_utils_python/delphi_utils/flash_eval/eval_day.py
index 660fca042..3c8012803 100644
--- a/_delphi_utils_python/delphi_utils/flash_eval/eval_day.py
+++ b/_delphi_utils_python/delphi_utils/flash_eval/eval_day.py
@@ -153,8 +153,7 @@ def output(evd_ranking, day, lag, signal, logger):
             p_text += f"\t{start_link}|*{index}*, {'{:.2f}'.format(value)}>\n"
         else:
             break
-    name = f"Signal: {signal} Lag: {lag}"
-    logger.info(name, payload=p_text)
+    logger.info("FLaSH: worth inspecting", signal=signal, lag=lag, payload=p_text)
 
 
 def evd_ranking_fn(ts_streams, EVD_max, EVD_min):
diff --git a/_delphi_utils_python/delphi_utils/logger.py b/_delphi_utils_python/delphi_utils/logger.py
index c0e4502a8..30fd78059 100644
--- a/_delphi_utils_python/delphi_utils/logger.py
+++ b/_delphi_utils_python/delphi_utils/logger.py
@@ -1,11 +1,11 @@
 """Structured logger utility for creating JSON logs.
 
-See the delphi_utils README.md for usage examples.
+To make our structured logging as useful as it can be, particularly within the context of how we use logs in Elastic,
+the `event` argument (typically the first unnamed arg) should be a static string (to make filtering easier),
+and each dynamic/varying value should be specified in an individual meaningfully- and consistently-named argument
+to the logger call (for use in filtering, thresholding, grouping, visualization, etc)
 
-The Delphi group uses two ~identical versions of this file.
-Try to keep them in sync with edits, for sanity.
-  https://github.com/cmu-delphi/covidcast-indicators/blob/main/_delphi_utils_python/delphi_utils/logger.py
-  https://github.com/cmu-delphi/delphi-epidata/blob/dev/src/common/logger.py
+See the delphi_utils README.md for usage examples.
 """
 
 import contextlib
diff --git a/_delphi_utils_python/delphi_utils/runner.py b/_delphi_utils_python/delphi_utils/runner.py
index abc28ba19..9083371aa 100644
--- a/_delphi_utils_python/delphi_utils/runner.py
+++ b/_delphi_utils_python/delphi_utils/runner.py
@@ -51,6 +51,7 @@ def run_indicator_pipeline(indicator_fn:  Callable[[Params], None],
 
     #Get version and indicator name for startup
     ind_name = indicator_fn.__module__.replace(".run", "")
+
     #Check for version.cfg in indicator directory
     if os.path.exists("version.cfg"):
         with open("version.cfg") as ver_file:
@@ -59,9 +60,15 @@ def run_indicator_pipeline(indicator_fn:  Callable[[Params], None],
                 if "current_version" in line:
                     current_version = str.strip(line)
                     current_version = current_version.replace("current_version = ", "")
-    #Logging - Starting Indicator
-        logger.info(f"Started {ind_name} with covidcast-indicators version {current_version}")
-    else: logger.info(f"Started {ind_name} without version.cfg")
+        logger.info(
+            "Started a covidcast-indicator",
+            indicator_name=ind_name,
+            current_version=current_version,
+        )
+    else:
+        logger.info(
+            "Started a covidcast-indicator without version.cfg", indicator_name=ind_name
+        )
 
     indicator_fn(params)
     validator = validator_fn(params)
@@ -77,8 +84,10 @@ def run_indicator_pipeline(indicator_fn:  Callable[[Params], None],
             break
         time.sleep(1)
     else:
-        logger.error(f"Flash step timed out ({timer} s), terminating",
-                elapsed_time_in_seconds = round(time.time() - start, 2))
+        logger.error(
+            "Flash step timed out, terminating",
+            elapsed_time_in_seconds=round(time.time() - start, 2),
+        )
         t1.terminate()
         t1.join()
     if validator:
diff --git a/_delphi_utils_python/tests/test_export.py b/_delphi_utils_python/tests/test_export.py
index 3e72f1d7f..c9c1f8483 100644
--- a/_delphi_utils_python/tests/test_export.py
+++ b/_delphi_utils_python/tests/test_export.py
@@ -323,12 +323,13 @@ def test_export_df_with_missingness(self, tmp_path):
 
     @mock.patch("delphi_utils.logger")
     def test_export_df_with_contradictory_missingness(self, mock_logger, tmp_path):
-
+        sensor = "test"
+        geo_res = "state"
         create_export_csv(
             df=self.DF3.copy(),
             export_dir=tmp_path,
-            geo_res="state",
-            sensor="test",
+            sensor=sensor,
+            geo_res=geo_res,
             logger=mock_logger
         )
         assert set(listdir(tmp_path)) == set(
@@ -339,8 +340,9 @@ def test_export_df_with_contradictory_missingness(self, mock_logger, tmp_path):
             ]
         )
         assert pd.read_csv(join(tmp_path, "20200315_state_test.csv")).size > 0
+        date_str = datetime.strftime(self.TIMES[0], "%Y-%m-%d")
         mock_logger.info.assert_called_once_with(
-            "Filtering contradictory missing code in test_None_2020-02-15."
+            "Filtering contradictory missing code", sensor=sensor, metric=None, date=date_str
         )
 
     def test_export_sort(self, tmp_path):
diff --git a/changehc/delphi_changehc/run.py b/changehc/delphi_changehc/run.py
index 92a03e6c5..9c15b221d 100644
--- a/changehc/delphi_changehc/run.py
+++ b/changehc/delphi_changehc/run.py
@@ -25,7 +25,7 @@ def retrieve_files(params, filedate, logger):
     if files["denom"] is None:
 
         ## download recent files from FTP server
-        logger.info("downloading recent files through SFTP")
+        logger.info("Downloading recent files through SFTP")
         download_counts(filedate, params["indicator"]["input_cache_dir"], params["indicator"]["ftp_conn"])
 
         denom_file = "%s/%s_Counts_Products_Denom.dat.gz" % (params["indicator"]["input_cache_dir"],filedate)
@@ -157,18 +157,20 @@ def run_module(params: Dict[str, Dict[str, Any]]):
 
     startdate, enddate = process_dates(params, startdate_dt, enddate_dt)
 
-    logger.info("generating signal and exporting to CSV",
-        first_sensor_date = startdate,
-        last_sensor_date = enddate,
-        drop_date = dropdate,
-        n_backfill_days = n_backfill_days,
-        n_waiting_days = n_waiting_days,
-        geos = params["indicator"]["geos"],
-        export_dir = params["common"]["export_dir"],
-        parallel = params["indicator"]["parallel"],
-        weekday = params["indicator"]["weekday"],
-        types = params["indicator"]["types"],
-        se = params["indicator"]["se"])
+    logger.info(
+        "Generating signal and exporting to CSV",
+        first_sensor_date=startdate,
+        last_sensor_date=enddate,
+        drop_date=dropdate,
+        n_backfill_days=n_backfill_days,
+        n_waiting_days=n_waiting_days,
+        geos=params["indicator"]["geos"],
+        export_dir=params["common"]["export_dir"],
+        parallel=params["indicator"]["parallel"],
+        weekday=params["indicator"]["weekday"],
+        types=params["indicator"]["types"],
+        se=params["indicator"]["se"],
+    )
 
     ## start generating
     stats = []
@@ -176,9 +178,9 @@ def run_module(params: Dict[str, Dict[str, Any]]):
         for numtype in params["indicator"]["types"]:
             for weekday in params["indicator"]["weekday"]:
                 if weekday:
-                    logger.info("starting weekday adj", geo = geo, numtype = numtype)
+                    logger.info("Starting weekday adj", geo_type=geo, numtype=numtype)
                 else:
-                    logger.info("starting no adj", geo = geo, numtype = numtype)
+                    logger.info("Starting no adj", geo_type=geo, numtype=numtype)
                 su_inst = CHCSensorUpdater(
                     startdate,
                     enddate,
@@ -211,7 +213,7 @@ def run_module(params: Dict[str, Dict[str, Any]]):
                 )
                 stats.extend(more_stats)
 
-            logger.info("finished processing", geo = geo)
+            logger.info("Finished processing", geo_type=geo)
 
     elapsed_time_in_seconds = round(time.time() - start_time, 2)
     min_max_date = stats and min(s[0] for s in stats)
diff --git a/changehc/delphi_changehc/sensor.py b/changehc/delphi_changehc/sensor.py
index 0449f07df..9a1fd29e0 100644
--- a/changehc/delphi_changehc/sensor.py
+++ b/changehc/delphi_changehc/sensor.py
@@ -118,10 +118,10 @@ def fit(y_data, first_sensor_date, geo_id, logger, num_col="num", den_col="den")
         se_valid = valid_rates.eval('sqrt(rate * (1 - rate) / den)')
         rate_data['se'] = se_valid
 
-        logger.debug("{0}: {1:.3f},[{2:.3f}]".format(
-            geo_id, rate_data['rate'][-1], rate_data['se'][-1]
-        ))
-        return {"geo_id": geo_id,
-                "rate": 100 * rate_data['rate'],
-                "se": 100 * rate_data['se'],
-                "incl": include}
+        logger.debug(
+            ".fit() DEBUG - last rate/se for geo",
+            geo_value=geo_id,
+            value=rate_data["rate"][-1],
+            se=rate_data["se"][-1],
+        )
+        return {"geo_id": geo_id, "rate": 100 * rate_data["rate"], "se": 100 * rate_data["se"], "incl": include}
diff --git a/changehc/delphi_changehc/update_sensor.py b/changehc/delphi_changehc/update_sensor.py
index edae85517..7c78dc020 100644
--- a/changehc/delphi_changehc/update_sensor.py
+++ b/changehc/delphi_changehc/update_sensor.py
@@ -41,7 +41,7 @@ def write_to_csv(df, geo_level, write_se, day_shift, out_name, logger, output_pa
     assert df[suspicious_se_mask].empty, " se contains suspiciously large values"
     assert not df["se"].isna().any(), " se contains nan values"
     if write_se:
-        logger.info("========= WARNING: WRITING SEs TO {0} =========".format(out_name))
+        logger.info("WARNING: WRITING SEs", filename=out_name)
     else:
         df["se"] = np.nan
 
@@ -49,9 +49,7 @@ def write_to_csv(df, geo_level, write_se, day_shift, out_name, logger, output_pa
     suspicious_val_mask = df["val"].gt(90)
     if not df[suspicious_val_mask].empty:
         for geo in df.loc[suspicious_val_mask, "geo_id"]:
-            logger.warning("value suspiciously high, {0}: {1}".format(
-                geo, out_name
-            ))
+            logger.warning("Value suspiciously high", geo_value=geo, filename=out_name)
 
     dates = create_export_csv(
         df,
@@ -62,10 +60,8 @@ def write_to_csv(df, geo_level, write_se, day_shift, out_name, logger, output_pa
         sensor=out_name,
         write_empty_days=True
     )
-    logger.debug("wrote {0} rows for {1} {2}".format(
-        df.size, df["geo_id"].unique().size, geo_level
-    ))
-    logger.debug("wrote files to {0}".format(output_path))
+    logger.debug("Wrote rows", num_rows=df.size, geo_type=geo_level, num_geo_ids=df["geo_id"].unique().size)
+    logger.debug("Wrote files", export_dir=output_path)
     return dates
 
 
@@ -148,8 +144,9 @@ def geo_reindex(self, data):
         geo = self.geo
         gmpr = GeoMapper()
         if geo not in {"county", "state", "msa", "hrr", "nation", "hhs"}:
-            self.logger.error("{0} is invalid, pick one of 'county', "
-                          "'state', 'msa', 'hrr', 'hss','nation'".format(geo))
+            self.logger.error(
+                "Geo is invalid, pick one of 'county', " "'state', 'msa', 'hrr', 'hss','nation'", geo_type=geo
+            )
             return False
         if geo == "county":
             data_frame = gmpr.fips_to_megacounty(data,
@@ -224,7 +221,7 @@ def update_sensor(self,
                 dfs.append(res)
         else:
             n_cpu = min(10, cpu_count())
-            self.logger.debug("starting pool with {0} workers".format(n_cpu))
+            self.logger.debug("Starting pool", n_workers=n_cpu)
             with Pool(n_cpu) as pool:
                 pool_results = []
                 for geo_id, sub_data in data_frame.groupby(level=0,as_index=False):
diff --git a/changehc/tests/test_update_sensor.py b/changehc/tests/test_update_sensor.py
index 7ef25a608..d2e7ee2f3 100644
--- a/changehc/tests/test_update_sensor.py
+++ b/changehc/tests/test_update_sensor.py
@@ -9,6 +9,7 @@
 import pandas as pd
 import numpy as np
 from boto3 import Session
+from delphi_utils import get_structured_logger
 from moto import mock_s3
 import pytest
 
@@ -28,7 +29,7 @@
 DENOM_FILEPATH = PARAMS["indicator"]["input_denom_file"]
 DROP_DATE = pd.to_datetime(PARAMS["indicator"]["drop_date"])
 OUTPATH="test_data/"
-TEST_LOGGER = logging.getLogger()
+TEST_LOGGER = get_structured_logger()
 
 class TestCHCSensorUpdater:
     """Tests for updating the sensors."""
diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py
index 2ce093488..ee6e98286 100644
--- a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py
+++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py
@@ -57,7 +57,7 @@ def download(ftp_credentials, out_path, logger):
     """Pull the latest raw files."""
     current_time = datetime.datetime.now()
     seconds_in_day = 24 * 60 * 60
-    logger.info("starting download", time=current_time)
+    logger.info("Starting download")
 
     # open client
     client = paramiko.SSHClient()
diff --git a/claims_hosp/delphi_claims_hosp/indicator.py b/claims_hosp/delphi_claims_hosp/indicator.py
index 4ad3ef7df..c5ac4e886 100644
--- a/claims_hosp/delphi_claims_hosp/indicator.py
+++ b/claims_hosp/delphi_claims_hosp/indicator.py
@@ -143,7 +143,10 @@ def fit(y_data, first_date, geo_id, num_col="num", den_col="den"):
         se_valid = valid_rates.eval('sqrt(rate * (1 - rate) / den)')
         rate_data['se'] = se_valid
 
-        logging.debug("%s: %05.3f, [%05.3f]",
-                      geo_id, rate_data['rate'][-1], rate_data['se'][-1])
-        return {"geo_id": geo_id, "rate": 100 * rate_data['rate'],
-                "se": 100 * rate_data['se'], "incl": include}
+        logging.debug(
+            ".fit() DEBUG - last rate/se for geo",
+            geo_value=geo_id,
+            value=rate_data["rate"][-1],
+            se=rate_data["se"][-1],
+        )
+        return {"geo_id": geo_id, "rate": 100 * rate_data["rate"], "se": 100 * rate_data["se"], "incl": include}
diff --git a/claims_hosp/delphi_claims_hosp/modify_claims_drops.py b/claims_hosp/delphi_claims_hosp/modify_claims_drops.py
index 0ab93ebcc..19a962884 100644
--- a/claims_hosp/delphi_claims_hosp/modify_claims_drops.py
+++ b/claims_hosp/delphi_claims_hosp/modify_claims_drops.py
@@ -57,5 +57,5 @@ def modify_and_write(data_path, logger, test_mode=False):
             dfs_list.append(dfs)
         else:
             dfs.to_csv(out_path, index=False)
-            logger.info(f"Wrote {out_path}")
+            logger.info("Wrote modified csv", filename=out_path)
     return files, dfs_list
diff --git a/claims_hosp/delphi_claims_hosp/run.py b/claims_hosp/delphi_claims_hosp/run.py
index 53c4cd33b..a9752072c 100644
--- a/claims_hosp/delphi_claims_hosp/run.py
+++ b/claims_hosp/delphi_claims_hosp/run.py
@@ -116,9 +116,9 @@ def run_module(params):
     for geo in params["indicator"]["geos"]:
         for weekday in params["indicator"]["weekday"]:
             if weekday:
-                logger.info("starting weekday adj", geo = geo)
+                logger.info("Starting weekday adj", geo_type=geo)
             else:
-                logger.info("starting no weekday adj", geo =  geo)
+                logger.info("Starting no weekday adj", geo_type=geo)
 
             signal_name = Config.signal_weekday_name if weekday else Config.signal_name
             if params["indicator"]["write_se"]:
@@ -126,7 +126,7 @@ def run_module(params):
                     "supply obfuscated prefix in params.json"
                 signal_name = params["indicator"]["obfuscated_prefix"] + "_" + signal_name
 
-            logger.info("Updating signal name", signal_name = signal_name)
+            logger.info("Updating signal name", signal=signal_name)
             updater = ClaimsHospIndicatorUpdater(
                 startdate,
                 enddate,
@@ -135,16 +135,16 @@ def run_module(params):
                 params["indicator"]["parallel"],
                 weekday,
                 params["indicator"]["write_se"],
-                signal_name
+                signal_name,
+                logger,
             )
             updater.update_indicator(
                 claims_file,
                 params["common"]["export_dir"],
-                logger,
             )
             max_dates.append(updater.output_dates[-1])
             n_csv_export.append(len(updater.output_dates))
-        logger.info("finished updating", geo = geo)
+        logger.info("Finished updating", geo_type=geo)
 
     # Remove all the raw files
     for fn in os.listdir(params["indicator"]["input_dir"]):
diff --git a/claims_hosp/delphi_claims_hosp/update_indicator.py b/claims_hosp/delphi_claims_hosp/update_indicator.py
index df3f3308f..5ba8ddd22 100644
--- a/claims_hosp/delphi_claims_hosp/update_indicator.py
+++ b/claims_hosp/delphi_claims_hosp/update_indicator.py
@@ -7,7 +7,6 @@
 """
 
 # standard packages
-import logging
 from multiprocessing import Pool, cpu_count
 
 # third party
@@ -28,8 +27,7 @@ class ClaimsHospIndicatorUpdater:
     # pylint: disable=too-many-instance-attributes, too-many-arguments
     # all variables are used
 
-    def __init__(self, startdate, enddate, dropdate, geo, parallel, weekday,
-                 write_se, signal_name):
+    def __init__(self, startdate, enddate, dropdate, geo, parallel, weekday, write_se, signal_name, logger):
         """
         Initialize updater for the claims-based hospitalization indicator.
 
@@ -53,6 +51,7 @@ def __init__(self, startdate, enddate, dropdate, geo, parallel, weekday,
         # init in shift_dates, declared here for pylint
         self.burnindate, self.fit_dates, self.burn_in_dates, self.output_dates = \
             [None] * 4
+        self.logger = logger
 
         assert (
                 self.startdate > (Config.FIRST_DATA_DATE + Config.BURN_IN_PERIOD)
@@ -114,9 +113,9 @@ def geo_reindex(self, data):
         elif self.geo == "hrr":
             data_frame = data  # data is already adjusted in aggregation step above
         else:
-            logging.error(
-                "%s is invalid, pick one of 'county', 'state', 'msa', 'hrr', 'hhs', nation'",
-                self.geo)
+            self.logger.error(
+                "Geo is invalid, pick one of 'county', 'state', 'msa', 'hrr', 'hhs', nation'", geo_type=self.geo
+            )
             return False
 
         unique_geo_ids = pd.unique(data_frame[self.geo])
@@ -133,7 +132,7 @@ def geo_reindex(self, data):
         data_frame.fillna(0, inplace=True)
         return data_frame
 
-    def update_indicator(self, input_filepath, outpath, logger):
+    def update_indicator(self, input_filepath, outpath):
         """
         Generate and output indicator values.
 
@@ -159,7 +158,7 @@ def update_indicator(self, input_filepath, outpath, logger):
                 ["num"],
                 Config.DATE_COL,
                 [1, 1e5],
-                logger,
+                self.logger,
             )
             if self.weekday
             else None
@@ -182,7 +181,7 @@ def update_indicator(self, input_filepath, outpath, logger):
                 valid_inds[geo_id] = np.array(res.loc[final_output_inds, "incl"])
         else:
             n_cpu = min(Config.MAX_CPU_POOL, cpu_count())
-            logging.debug("starting pool with %d workers", n_cpu)
+            self.logger.debug("Starting pool", n_workers=n_cpu)
             with Pool(n_cpu) as pool:
                 pool_results = []
                 for geo_id, sub_data in data_frame.groupby(level=0, as_index=False):
@@ -217,7 +216,7 @@ def update_indicator(self, input_filepath, outpath, logger):
         }
 
         self.write_to_csv(output_dict, outpath)
-        logging.debug("wrote files to %s", outpath)
+        self.logger.debug("Wrote files", export_dir=outpath)
 
     def write_to_csv(self, output_dict, output_path="./receiving"):
         """
@@ -229,8 +228,7 @@ def write_to_csv(self, output_dict, output_path="./receiving"):
 
         """
         if self.write_se:
-            logging.info("========= WARNING: WRITING SEs TO %s =========",
-                         self.signal_name)
+            self.logger.info("WARNING: WRITING SEs", signal=self.signal_name)
 
         geo_level = output_dict["geo_level"]
         dates = output_dict["dates"]
@@ -255,7 +253,7 @@ def write_to_csv(self, output_dict, output_path="./receiving"):
                         assert not np.isnan(val), "value for included value is nan"
                         assert not np.isnan(se), "se for included rate is nan"
                         if val > 90:
-                            logging.warning("value suspicious, %s: %d", geo_id, val)
+                            self.logger.warning("Value suspicious", geo_type=geo_level, geo_value=geo_id, value=val)
                         assert se < 5, f"se suspicious, {geo_id}: {se}"
                         if self.write_se:
                             assert val > 0 and se > 0, "p=0, std_err=0 invalid"
@@ -267,4 +265,4 @@ def write_to_csv(self, output_dict, output_path="./receiving"):
                                 "%s,%f,%s,%s,%s\n" % (geo_id, val, "NA", "NA", "NA"))
                         out_n += 1
 
-        logging.debug("wrote %d rows for %d %s", out_n, len(geo_ids), geo_level)
+        self.logger.debug("Wrote rows", num_rows=out_n, geo_type=geo_level, num_geo_ids=len(geo_ids))
diff --git a/claims_hosp/tests/test_update_indicator.py b/claims_hosp/tests/test_update_indicator.py
index 1471be655..5ca527287 100644
--- a/claims_hosp/tests/test_update_indicator.py
+++ b/claims_hosp/tests/test_update_indicator.py
@@ -51,7 +51,8 @@ def test_shift_dates(self):
             self.parallel,
             self.weekday,
             self.write_se,
-            Config.signal_name
+            Config.signal_name,
+            TEST_LOGGER
         )
         ## Test init
         assert updater.startdate.month == 2
@@ -72,7 +73,8 @@ def test_geo_reindex(self):
             self.parallel,
             self.weekday,
             self.write_se,
-            Config.signal_name
+            Config.signal_name,
+            TEST_LOGGER
         )
         updater.shift_dates()
         data_frame = updater.geo_reindex(self.small_test_data.reset_index())
@@ -90,13 +92,13 @@ def test_update_indicator(self):
                 self.parallel,
                 self.weekday,
                 self.write_se,
-                Config.signal_name
+                Config.signal_name,
+                TEST_LOGGER
             )
 
             updater.update_indicator(
                 DATA_FILEPATH,
-                td.name,
-                TEST_LOGGER
+                td.name
             )
 
             assert len(os.listdir(td.name)) == len(
@@ -112,7 +114,8 @@ def test_write_to_csv_results(self):
             self.parallel,
             self.weekday,
             self.write_se,
-            Config.signal_name
+            Config.signal_name,
+            TEST_LOGGER
         )
 
         res0 = {
@@ -192,7 +195,8 @@ def test_write_to_csv_with_se_results(self):
             self.parallel,
             True,
             True,
-            signal_name
+            signal_name,
+            TEST_LOGGER
         )
 
         res0 = {
@@ -243,7 +247,8 @@ def test_write_to_csv_wrong_results(self):
             self.parallel,
             self.weekday,
             self.write_se,
-            Config.signal_name
+            Config.signal_name,
+            TEST_LOGGER
         )
 
         res0 = {
diff --git a/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py b/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py
index 9d51768be..6cb42364a 100644
--- a/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py
+++ b/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py
@@ -59,7 +59,7 @@ def download(ftp_credentials, out_path, logger, issue_date=None):
     else:
         current_time = datetime.datetime.strptime(issue_date, "%Y-%m-%d").replace(hour=23, minute=59, second=59)
 
-    logger.info("starting download", time=current_time)
+    logger.info("Starting download")
     seconds_in_day = 24 * 60 * 60
 
     # open client
diff --git a/doctor_visits/delphi_doctor_visits/modify_claims_drops.py b/doctor_visits/delphi_doctor_visits/modify_claims_drops.py
index daed93d58..3be9393e4 100644
--- a/doctor_visits/delphi_doctor_visits/modify_claims_drops.py
+++ b/doctor_visits/delphi_doctor_visits/modify_claims_drops.py
@@ -48,5 +48,5 @@ def modify_and_write(f, logger, test_mode=False):
 
     if not test_mode:
         dfs.to_csv(out_path, index=False)
-        logger.info(f"Wrote {out_path}")
+        logger.info("Wrote modified csv", filename=out_path)
     return dfs
diff --git a/doctor_visits/delphi_doctor_visits/patch.py b/doctor_visits/delphi_doctor_visits/patch.py
index 32b6d308f..32c62dd6b 100644
--- a/doctor_visits/delphi_doctor_visits/patch.py
+++ b/doctor_visits/delphi_doctor_visits/patch.py
@@ -45,16 +45,19 @@ def patch():
     start_issue = datetime.strptime(params["patch"]["start_issue"], "%Y-%m-%d")
     end_issue = datetime.strptime(params["patch"]["end_issue"], "%Y-%m-%d")
 
-    logger.info(f"""Start patching {params["patch"]["patch_dir"]}""")
-    logger.info(f"""Start issue: {start_issue.strftime("%Y-%m-%d")}""")
-    logger.info(f"""End issue: {end_issue.strftime("%Y-%m-%d")}""")
+    logger.info(
+        "Starting patching",
+        patch_directory=params["patch"]["patch_dir"],
+        start_issue=start_issue.strftime("%Y-%m-%d"),
+        end_issue=end_issue.strftime("%Y-%m-%d"),
+    )
 
     makedirs(params["patch"]["patch_dir"], exist_ok=True)
 
     current_issue = start_issue
 
     while current_issue <= end_issue:
-        logger.info(f"""Running issue {current_issue.strftime("%Y-%m-%d")}""")
+        logger.info("Running issue", issue_date=current_issue.strftime("%Y-%m-%d"))
 
         params["patch"]["current_issue"] = current_issue.strftime("%Y-%m-%d")
 
diff --git a/doctor_visits/delphi_doctor_visits/run.py b/doctor_visits/delphi_doctor_visits/run.py
index 3c941534a..2dccffc8c 100644
--- a/doctor_visits/delphi_doctor_visits/run.py
+++ b/doctor_visits/delphi_doctor_visits/run.py
@@ -88,32 +88,33 @@ def run_module(params, logger=None):  # pylint: disable=too-many-statements
     startdate_dt = enddate_dt - timedelta(days=n_backfill_days)
     enddate = str(enddate_dt.date())
     startdate = str(startdate_dt.date())
-    logger.info("drop date:\t\t%s", dropdate)
-    logger.info("first sensor date:\t%s", startdate)
-    logger.info("last sensor date:\t%s", enddate)
-    logger.info("n_backfill_days:\t%s", n_backfill_days)
-    logger.info("n_waiting_days:\t%s", n_waiting_days)
+
+    logger.info(
+        "Using params",
+        startdate=startdate,
+        enddate=enddate,
+        dropdate=dropdate,
+        n_backfill_days=n_backfill_days,
+        n_waiting_days=n_waiting_days,
+        export_dir=export_dir,
+        parallel=params["indicator"]["parallel"],
+        weekday=params["indicator"]["weekday"],
+        write_se=se,
+        prefix=prefix,
+    )
 
     ## geographies
     geos = ["state", "msa", "hrr", "county", "hhs", "nation"]
 
-
-    ## print out other vars
-    logger.info("outpath:\t\t%s", export_dir)
-    logger.info("parallel:\t\t%s", params["indicator"]["parallel"])
-    logger.info("weekday:\t\t%s", params["indicator"]["weekday"])
-    logger.info("write se:\t\t%s", se)
-    logger.info("obfuscated prefix:\t%s", prefix)
-
     max_dates = []
     n_csv_export = []
     ## start generating
     for geo in geos:
         for weekday in params["indicator"]["weekday"]:
             if weekday:
-                logger.info("starting %s, weekday adj", geo)
+                logger.info("Starting with weekday adj", geo_type=geo)
             else:
-                logger.info("starting %s, no adj", geo)
+                logger.info("Starting with no adj", geo_type=geo)
             sensor = update_sensor(
                 filepath=claims_file,
                 startdate=startdate,
@@ -137,8 +138,8 @@ def run_module(params, logger=None):  # pylint: disable=too-many-statements
             write_to_csv(sensor, geo, se, out_name, logger, export_dir)
             max_dates.append(sensor.date.max())
             n_csv_export.append(sensor.date.unique().shape[0])
-            logger.debug(f"wrote files to {export_dir}")
-        logger.info("finished updating", geo = geo)
+            logger.debug("Wrote files", export_dir=export_dir)
+        logger.info("Finished updating", geo_type=geo)
 
     # Remove all the raw files
     for fn in os.listdir(params["indicator"]["input_dir"]):
diff --git a/doctor_visits/delphi_doctor_visits/sensor.py b/doctor_visits/delphi_doctor_visits/sensor.py
index b5a645ea8..91faa20cf 100644
--- a/doctor_visits/delphi_doctor_visits/sensor.py
+++ b/doctor_visits/delphi_doctor_visits/sensor.py
@@ -239,7 +239,7 @@ def fit(y_data,
         se[include] = np.sqrt(
             np.divide((new_rates[include] * (1 - new_rates[include])), den[include]))
 
-        logger.debug(f"{geo_id}: {new_rates[-1]:.3f},[{se[-1]:.3f}]")
+        logger.debug(".fit() DEBUG - last rate/se for geo", geo_value=geo_id, value=new_rates[-1], se=se[-1])
 
         included_indices = [x for x in final_sensor_idxs if include[x]]
 
diff --git a/doctor_visits/delphi_doctor_visits/update_sensor.py b/doctor_visits/delphi_doctor_visits/update_sensor.py
index 125c0df18..4cac1e81c 100644
--- a/doctor_visits/delphi_doctor_visits/update_sensor.py
+++ b/doctor_visits/delphi_doctor_visits/update_sensor.py
@@ -34,7 +34,7 @@ def write_to_csv(output_df: pd.DataFrame, geo_level, se, out_name, logger, outpu
       output_path: outfile path to write the csv (default is current directory)
     """
     if se:
-        logger.info(f"========= WARNING: WRITING SEs TO {out_name} =========")
+        logger.info("WARNING: WRITING SEs", filename=out_name)
 
     out_n = 0
     for d in set(output_df["date"]):
@@ -64,7 +64,7 @@ def write_to_csv(output_df: pd.DataFrame, geo_level, se, out_name, logger, outpu
                     outfile.write(
                         "%s,%f,%s,%s,%s\n" % (geo_id, sensor, "NA", "NA", "NA"))
                 out_n += 1
-    logger.debug(f"wrote {out_n} rows for {geo_level}")
+    logger.debug("Wrote rows", num_rows=out_n, geo_type=geo_level)
 
 
 def update_sensor(
@@ -177,7 +177,7 @@ def update_sensor(
 
     else:
         n_cpu = min(10, cpu_count())
-        logger.debug(f"starting pool with {n_cpu} workers")
+        logger.debug("Starting pool", n_workers=n_cpu)
 
         with Pool(n_cpu) as pool:
             pool_results = []
diff --git a/google_symptoms/delphi_google_symptoms/date_utils.py b/google_symptoms/delphi_google_symptoms/date_utils.py
index ebfe5109a..2ad6244e9 100644
--- a/google_symptoms/delphi_google_symptoms/date_utils.py
+++ b/google_symptoms/delphi_google_symptoms/date_utils.py
@@ -98,7 +98,7 @@ def generate_num_export_days(params: Dict, logger) -> [int]:
                 expected_date_diff += global_max_expected_lag
 
             if latest_date_diff > expected_date_diff:
-                logger.info(f"Missing dates from: {to_datetime(min(gs_metadata.max_time)).date()}")
+                logger.info("Missing date", date=to_datetime(min(gs_metadata.max_time)).date())
 
             num_export_days = expected_date_diff
 
diff --git a/google_symptoms/delphi_google_symptoms/patch.py b/google_symptoms/delphi_google_symptoms/patch.py
index 01d099c4f..85df89394 100755
--- a/google_symptoms/delphi_google_symptoms/patch.py
+++ b/google_symptoms/delphi_google_symptoms/patch.py
@@ -58,16 +58,19 @@ def patch(params):
     issue_date = datetime.strptime(params["patch"]["start_issue"], "%Y-%m-%d")
     end_issue = datetime.strptime(params["patch"]["end_issue"], "%Y-%m-%d")
 
-    logger.info(f"""Start patching {params["patch"]["patch_dir"]}""")
-    logger.info(f"""Start issue: {issue_date.strftime("%Y-%m-%d")}""")
-    logger.info(f"""End issue: {end_issue.strftime("%Y-%m-%d")}""")
+    logger.info(
+        "Starting patching",
+        patch_directory=params["patch"]["patch_dir"],
+        start_issue=issue_date.strftime("%Y-%m-%d"),
+        end_issue=end_issue.strftime("%Y-%m-%d"),
+    )
 
     makedirs(params["patch"]["patch_dir"], exist_ok=True)
 
     patch_dates = generate_patch_dates(params)
 
     while issue_date <= end_issue:
-        logger.info(f"""Running issue {issue_date.strftime("%Y-%m-%d")}""")
+        logger.info("Running issue", issue_date=issue_date.strftime("%Y-%m-%d"))
 
         # Output dir setup
         current_issue_yyyymmdd = issue_date.strftime("%Y%m%d")
diff --git a/google_symptoms/delphi_google_symptoms/run.py b/google_symptoms/delphi_google_symptoms/run.py
index 8303a9a8a..8ad1d6d10 100644
--- a/google_symptoms/delphi_google_symptoms/run.py
+++ b/google_symptoms/delphi_google_symptoms/run.py
@@ -80,10 +80,8 @@ def run_module(params, logger=None):
         if len(df_pull) == 0:
             continue
         for metric, smoother in product(COMBINED_METRIC, SMOOTHERS):
-            logger.info("generating signal and exporting to CSV",
-                        geo_res=geo_res,
-                        metric=metric,
-                        smoother=smoother)
+            sensor_name = "_".join([smoother, "search"])
+            logger.info("Generating signal and exporting to CSV", geo_type=geo_res, signal=f"{metric}_{sensor_name}")
             df = df_pull
             df["val"] = df[metric].astype(float)
             df["val"] = df[["geo_id", "val"]].groupby(
@@ -94,9 +92,8 @@ def run_module(params, logger=None):
             # Drop early entries where data insufficient for smoothing
             df = df.loc[~df["val"].isnull(), :]
             df = df.reset_index()
-            sensor_name = "_".join([smoother, "search"])
             if len(df) == 0:
-                logger.info("No data for %s_%s_%s", geo_res, metric.lower(), sensor_name)
+                logger.info("No data for signal", geo_type=geo_res, signal=f"{metric}_{sensor_name}")
                 continue
             exported_csv_dates = create_export_csv(
                 df,
diff --git a/nssp/delphi_nssp/run.py b/nssp/delphi_nssp/run.py
index 931cd7c71..b22d03c20 100644
--- a/nssp/delphi_nssp/run.py
+++ b/nssp/delphi_nssp/run.py
@@ -90,7 +90,7 @@ def run_module(params):
         for geo in GEOS:
             df = df_pull.copy()
             df["val"] = df[signal]
-            logger.info("Generating signal and exporting to CSV", metric=signal)
+            logger.info("Generating signal and exporting to CSV", geo_type=geo, signal=signal)
             if geo == "nation":
                 df = df[df["geography"] == "United States"]
                 df["geo_id"] = "us"
diff --git a/nwss_wastewater/delphi_nwss/run.py b/nwss_wastewater/delphi_nwss/run.py
index 378849ba5..60bfc84c7 100644
--- a/nwss_wastewater/delphi_nwss/run.py
+++ b/nwss_wastewater/delphi_nwss/run.py
@@ -146,7 +146,7 @@ def run_module(params):
         df = generate_weights(df, sensor)
 
         for geo in GEOS:
-            logger.info("Generating signal and exporting to CSV", metric=sensor)
+            logger.info("Generating signal and exporting to CSV", geo_type=geo, signal=sensor)
             if geo == "nation":
                 agg_df = weighted_nation_sum(df, sensor)
             else:
diff --git a/quidel_covidtest/delphi_quidel_covidtest/pull.py b/quidel_covidtest/delphi_quidel_covidtest/pull.py
index d9f23f2ec..560f89456 100644
--- a/quidel_covidtest/delphi_quidel_covidtest/pull.py
+++ b/quidel_covidtest/delphi_quidel_covidtest/pull.py
@@ -56,7 +56,7 @@ def get_from_s3(start_date, end_date, bucket, logger):
     seen_files = set()
     for search_date in [start_date + timedelta(days=x) for x in range(n_days)]:
         if search_date in s3_files.keys():
-            logger.info(f"Pulling data received on {search_date.date()}")
+            logger.info("Pulling data received on date", search_date=search_date.date())
 
             # Fetch data received on the same day
             for fn in s3_files[search_date]:
@@ -110,11 +110,11 @@ def fix_date(df, logger):
     df.insert(2, "timestamp", df["TestDate"])
 
     mask = df["TestDate"] <= df["StorageDate"]
-    logger.info(f"Removing {((len(df) - np.sum(mask)) * 100 / len(df)):.2f}% of unusual data")
+    logger.info("Removing unusual data", percent=round((len(df) - np.sum(mask)) * 100 / len(df), 2))
     df = df[mask]
 
     mask = df["StorageDate"] - df["TestDate"] > pd.Timedelta(days=90)
-    logger.info(f"Fixing {(np.sum(mask) * 100 / len(df)):.2f}% of outdated data")
+    logger.info("Fixing outdated data", percent=round((np.sum(mask) * 100 / len(df)), 2))
     df["timestamp"].values[mask] = df["StorageDate"].values[mask]
     return df
 
diff --git a/quidel_covidtest/delphi_quidel_covidtest/run.py b/quidel_covidtest/delphi_quidel_covidtest/run.py
index a59e0c101..e6974b6aa 100644
--- a/quidel_covidtest/delphi_quidel_covidtest/run.py
+++ b/quidel_covidtest/delphi_quidel_covidtest/run.py
@@ -164,7 +164,7 @@ def run_module(params: Dict[str, Any]):
     n_cpu = min(8, cpu_count()) # for parallelization
     with pool_and_threadedlogger(logger, n_cpu) as (pool, threaded_logger):
         # for using loggers in multiple threads
-        logger.info("Parallelizing sensor generation", n_cpu=n_cpu)
+        logger.info("Parallelizing sensor generation", n_workers=n_cpu)
         pool_results = []
         for geo_res in NONPARENT_GEO_RESOLUTIONS:
             geo_data, res_key = geo_map(geo_res, data)

From 9edcc92c8739367c8565a805f51e8e1fdbf5f51e Mon Sep 17 00:00:00 2001
From: Brian Clark <clark.bg@gmail.com>
Date: Mon, 21 Oct 2024 09:21:59 -0400
Subject: [PATCH 11/13] Update path for pyenv shims

---
 Jenkinsfile | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1b9485ca5..3fcdc904a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -18,6 +18,10 @@ def deploy_production = [:]
 
 pipeline {
     agent any
+    environment {
+        // Set the PATH variable to include the pyenv shims directory.
+        PATH = "/var/lib/jenkins/.pyenv/shims:${env.PATH}"
+    }
     stages {
         stage('Build dev/feature branch') {
             when  {

From 1a03d5676f954553210ac07f7a9ae98a18d5ca70 Mon Sep 17 00:00:00 2001
From: Brian Clark <clark.bg@gmail.com>
Date: Mon, 21 Oct 2024 16:40:41 -0400
Subject: [PATCH 12/13] Update differ bucket access

---
 ansible/vault.yaml | 528 ++++++++++++++++++++++-----------------------
 1 file changed, 264 insertions(+), 264 deletions(-)

diff --git a/ansible/vault.yaml b/ansible/vault.yaml
index bf979841a..1a674592b 100644
--- a/ansible/vault.yaml
+++ b/ansible/vault.yaml
@@ -1,265 +1,265 @@
 $ANSIBLE_VAULT;1.1;AES256
-63313737356234663364633839353638393064663937643934333430643832363065646138333038
-3133663766613235613137383736383030636236323166370a366235353833306336336432376333
-61336662656336336530326463663761366232393063326365366561386636636165396430623165
-6131643963386231650a356137646637333266613966343631656136303438303932326638363038
-30653364343436383433663639393633356633393836613263663537366335316130323365653736
-31353831343864383932393661383431346337353863643063636632313666613562306237623930
-63633061343263626139613337376333653562653432363763306535663735373433336636376437
-37646265373831613461363838303635306662633637643535623961376564653265663633656537
-66663134383638636138386137626133613964363333643133356365356237313935363531373066
-61626337396232316535376339306162316266366261383630623466306634643533326166386234
-65323262363337343261633638336364333230343035653032383031333431623032643432353136
-66653865623931313266326161343064383938636435306636383033313731323762383462303238
-34643562306264316130316266623838613361393462663361386234626661353838363036323937
-31653432623236303263643739326561306131316337313030636563363838353638353665343339
-38383938333164383337623634363430366232666231323539373239363266623339306434313162
-66393238663031376231386431666132643139383731323535313839643864633239346364303333
-61623238303038663833396663633963323863636139616631373036376337396262643862633939
-61646662653238396532613962376237623561313461333061356663343664306563386539376633
-32616433333163643066313330666263653939303932653835303562623061363833636435643534
-62336561306335346664333833306335323364613039653263373334616134313962613630386262
-63306630333739616137346338383964346365663836353166653236636663366539396431323032
-33366362346462303865623937363032363233316530656534383164393138323565306139306461
-35656266633636393631353435386337333039303535343233353134333839376163316262316533
-62313764306330316339623033636436313539326664353339326635613462316362333661363635
-66326565636439353466656663303663636237333965393961656637623933636366653562663564
-38373463626139313234623333306261666239303135333965656537393338323763666131366330
-38303435353765366162373430393231656139316238613839353432306231343130373038376135
-33396232636131653534666164313131333861336130636664643730393631626361353739303835
-31666132643830376335343938376338666538663865613132313130323736616237323134333031
-35656663346464383233646362663736633230613437643834383833633637656161303865343362
-64303636616262383865343332353139633432626565666233656436653761626563653639613334
-64313030643363323665613965353565393762323065323735306337373833373035396430376137
-62383063626430313532653261616230336164666461613966623032323965623962373830626236
-37356361326562613634376261336436346535333831643764343932356334616535343364316563
-64333731383831323165663530613736323461643836363438343134633635336431376664366364
-37326565613631613333313632666538383966363739666338626431643734636565616665653938
-37623839313030653733626263656666363362373135623562653462363330366136373938663532
-30343436303863636636353931656562303565626631323238356463373330393161373932373965
-34636638633764616430336131363734323932656135343032636564383231363731663963666663
-38376263333938643933363336373637396237303961653062343866363632663532336434643063
-63353739343033396130643638396365336332626439313231396361383736373036393838623532
-35383566333238626131343530313463326631313462333462396233613438333365613932663237
-35376232383238323636343335643236656135333766386565326337643466363064616131616430
-33643131363639363164316438656364386532626134386666613533643831326162363961666332
-62303938666565393938393133363734613365303136623638613532333033346465633830333164
-30333363376338373063373338393435626566356338393036303939353236633339373538613661
-63653064616338313135373238383432363138356665386638343434326337343862313431333535
-33633761333836376238376530636236613165353266363433663065633038626532343462353164
-61396530373532343831303331316236373834363538333632323664633363636133373438376234
-34343136363734333963383865646361653233393132373938633930366462383632333839353861
-63353030373137633161396165636366613337646461353265616263303663393636303231386234
-39356263323831313439333630376365343530336539663139373933336265303834623734656339
-35326239333163346335316138303831303063613839353164663563363737303834343132393237
-30376132303862386137626333653539303133366233303335356636303737353661306163313165
-35623865336130633366633131343838343830613537626539306566306439393737343438386162
-65393937323337306439663435346132386530353564333730626533373537646461646339323530
-31613033376539653363636332346238316632306463343436346239633364306638663464396639
-30653661616636663762353461366330613337643531336239346163306234313330663435336439
-61333334393937303933393965663030656232336464313135323164323030656134663030346366
-63613638623666363936613065643137636662306662396332643362333439393732326263623461
-37396135353033353832393763356439643566373738336438623234396365613136616432633636
-31363735393664303039333233313636326564363963616365646337373065363937306164643239
-38636131633563616339343734313836613462646636323163336136623034613965633433633035
-33396636323662343638616463303731306665616232643161613937356530303562383033626365
-64393835636365383432653330633538336435373336623833626337363264373665383530663538
-62316266326634376133666334336563656662666536353434306232373465623835346233316664
-63643966613363366363363265626435626132346532333864623834323061383235623034316139
-31386163623133366365353766373764323833363035333835383163656363386338386433323365
-38616437353530643739306365386134376364623665313965656533323334306438393031623434
-62373632633636663832353465353666333362663266623633376337393330353635386533343235
-63633938343062336436356334616633393331666562313165666664343133393935633465393739
-66633031323531346630343265663933666338623765663163336435343632303132313139656637
-64653030633938326439336331393763316161326464303231353066626464633836323431303666
-63316230623539333663623961393434393239373132363935326635383930626436636135386663
-61343433666365336638393639323038333032363462663734353031376463383531393661396333
-32386666316230613463333964323332373765333962383764613931643266313839333639383964
-34663761336166366532383138623230613037646165363039613138316464306135366366373136
-32353937663863333738356166623836303863343665353564326562313937333965646132626664
-62323434313964636564393432636534396261346265353336636564386338363137386632306661
-32653938393831646136613464663532313531636331633264303230306134623431333564666130
-66363330326661393339373235383530656562316531373666656638343038336164323363343637
-61653131313462356538656232643237623038666664663435363134363162306538383930373637
-63633539623265393137393332336463653132363661356365656237663461613764663837323336
-34613732363466353836363064383661626431363061613430616562396161366430363362336338
-35626265336562336134633538326365383365383766663731633738323430633365663266393061
-63396365303462386665343666346133303661616435663566303634363833303663383464623237
-30393364326333393932303462386435616163366165646463383765396132663166316666656462
-36346266636539663336646231366463353839363639613765313663303232633163356565643565
-61633163373335316362343835663364626336656236373361373830303463363965333432363134
-37333961636163373963613263353039636266376265333632363163363334353563323533663064
-33323234623035336232326238623838383533616533626665643133636233313062343137376263
-36653762666562633837356362383662663338616635646535393139356336346639326264643835
-61356531316366373038666166643434656633623635333238303831656338336539313931383762
-63376434373435646562623934346437316465303565313564613365663533303665363662653039
-61343363383438633236643664396533373134633963386638396236386561373161376365303832
-33333563306662623137633231613566353361643735666364653233666136313230633338323764
-65373334633836623666396637363166323631383835356666336461636433653464366138343630
-33323064353661633131303830613534636164643432303165376437346161373739333633616533
-63356331373061663463623435653632353433316261343562303837336135616636353864313432
-34643362303063363861646639336236623533636463306137393034633837643331653739643662
-65613763373665343830363063643738326439306164633533376339366265336466313432643136
-36386561343862376463306438326539393163356165313936323939323831343339393433643135
-38303862393932306161383236663466653933353932616537383965316464643762363331336137
-35663533316466613862373634323535663434303563386232306339613239633732346238323163
-61313330356631396135313765646435336633643038653835363436386363373138666266656162
-39363833396165323236376635626130623034336139336233333931323933306664333533653361
-35646139666131643430353731326137363135316236383833373631396266666463626464353365
-65666630633437343235336561366566366139393965383663623534643333383637646662313134
-30363161613464656632313536373631633166313162303761303835343338643366306330343232
-65656238346132383134626564333336306463643962383530343064343263643435363638366637
-62393337313732353230366536383434386332383837303161366166663430656335333733653738
-34386637353132323939323162323734643936666563663037353832666330366538333138653536
-62343462353835333232376666653538346564343566303135333434303963393032623564303034
-62653732666135383666623061646339366563316564613031303639323732666362326630643931
-35333436313236666563376131336233396336626161313863393962386638656634346137646530
-66376563336432633331643939623834353162613030326163383565396334383839643661363737
-36666363636237366537376565656465663834646665333930653132393461636334303334396161
-30363332303938336462646632303435643635363163333431343032616432646233353463396632
-32303938623263366634376536353764616361393430393361316263336239333633396666666566
-62633032383131636162333336306139356239313732383939323662386333363635663634616633
-61393262616162623966393264323466343738303566653862353939303637303039363136623662
-66336166326336373962306138373137313731656330373036656662363964646665386131656334
-36353432636238636161633131363161316438306430356233383232303163633764633736343264
-62346664663362656634666634353939663937613863383037333861386135353338613461663466
-36386463373938383637653733333637656362663061396466373563653861386630346330626261
-33656164363362306433356137346335363830306461303431333461393238373838336235666463
-33316235363338383435303939656331636639623733636566396536383461353534613331356139
-36333165303562313236623265613136653935346237343361636136343835363162656636313564
-63373063336434376232393662316634343734333534363537373637636264643666383835303536
-65653534393839623063396433663630363263626232613065663031333235636333643062313065
-30333330636339313032653431646639373465633633386166646139343461643964356263623738
-34356661663339386135383732643039613034326436643334656564646564326333323239333235
-38383562353263623265623230323333623536323139366462316665626537656132356561616361
-39313639333638656565393531646564643038336564623635343438376533343732663635653031
-30633231383431663834313066396362636238386332633665663661633733373836656335313239
-33386638393935346361376637663539663832393763343662643363623165316331336463663134
-62633832336236633831653837643764323031663734376262373866643036613931306633343065
-34383637646534373135306230376163356461646136393463623866303834333636633536336263
-33373066366333663063623638643035343161353633633631643461383332353263303962303432
-33303737336137643463343333633065643237333539383563376662373833623931333366373833
-65653733653363613561316365646363353535323838386133646532353364306639646561323937
-66396663383934616665653833646661333630303737666239383365363530393430323439353030
-36393064393163663234393665393233393534376236633139626662666330666461633437646466
-35346262663265386236366431623937396330616230306533616266616265653638643732653164
-39336232376239656134356363636332343631653133653231366338373535313363646430653137
-34653435396338306436346664383932336130393966643561373062396438363839363938313036
-38303761616630303265313665623535633032626333323932386535376166353534663262653130
-39313664616166383530343463633561306137313162346335356562346636383239323765343962
-35393462653034386662306438363533333865343634343565653164613566313331653832613035
-31646462363465316133653433653165633530643666633031643133656638323132653335626431
-37613631366332663165306264373736353532376431326131623764353238623762393763373365
-33663738626263663936373138613764386437656432303135656265636365353834616465633062
-66306435396639663533633465396664303936313936613330646564613630303131373437313538
-31306637643434303635316563323662643433643531626332613437353261313963373933623062
-37303331366561386233353665346236643363326537616532306236646533646163343862386333
-33343335373864636232323538356663343230323362346533323865383630613666366364383964
-65633434353763323033386231623737363033376438616162333339643463323730333435663566
-31323139326465393261653335366330313936383939313735353133393938313066636533336664
-31346166613434326665333866633762663338636335373965613435396539363834326338386564
-37626666653961376265343138323934373234636161373037656530643832663764346632623734
-38306461646231636666623166323334323765373639343665663537356564356332366632623434
-62353666306635393237313838313739313330623466376638653766613734663964666334313834
-32353137333564343334613937353766663532323063663233623865656439323864393838643038
-30613164356138316534643166633834663531653664306563333834636632346635666662323435
-62366662353930373637613032316639323836626166663766396461626130393031383331343734
-61633165663461353336303761613637323435383033363437396531613332323037346562643835
-38643938616436303062323763396338353365653738323838353865313533666639646462633933
-33323235666334623137363331313565366631633533643834323530333462386539626335303935
-37323161613762333932303465656238333435333162383338396565313665373537336439336539
-63326133353762346366313963383636666438633439346436306661366563373934663034653037
-32656463363338656562613332613566323531353964613139376530303437396139656530396166
-61626561663836643333653736316530393566666364633563633336666431616531663233343065
-39353036336566613934316434306261626363636536643265393230623630316562636265373766
-39396166373565616464386234363639323839346234303063666239323337366330663538303532
-32366338323263323530633762336438303037623330316666383836356630346235333065383263
-35333734623461373331393436666164656433646361323638353430373739656338306438333231
-31623632356231616663613734343965326662636330646533363538303562323433643430333565
-61356639383464393538666134396634366264333338396435363336656334376530343162373464
-66396336363136666335613737666330646536663364643733353761306534363863656461383331
-37373061333138373331356535316338626238326134626330356234376133316163663264643333
-38646131323063343032613765663232323164303338303634336366353238663532356335376638
-64323165663261363431646532383037666262626638373131653663303931383535636236316531
-36343736363431316661383031626633343334653164393231303339303564373665623434326639
-64383062373566313562653836326665623531316531303334306431383466313763636338383035
-30383762336666343639623138623961366365303631656438636466613663653537623635636262
-32616662303765343237373936623664383035333437663436333231353239663966343764303761
-30663837356435383333333237393664386631313337366437333735626136313561336465303934
-61623934346330626435303034366636306364376466623364366433326539653466353232616465
-61393632336463383262663235323665353931613065326565656336396261356235303737383662
-62333035316265646533653061363266303037653766366262326631376466343932313530653238
-30323165633338386637333865313761646633633735393238643464653638343934376435343962
-34323934383038323439626239336332653330366263343961356438373939373065393036633536
-64643839636261633234666663303063386438303132633166636631356166316535666132356161
-36333535663235663562383935663463336638383565326630393337663930376632666364353239
-61336233393166623937386666313464626633353430663064366631323831333739356261393837
-35333631383034313463336263353135656234623535303233353465316166623132333063386366
-35393234646139323132326163396632353763353262363139613562346266656266626435336330
-30616566353163326336386266666136643233393662643562376337366433306139393438373130
-37383565643931646632323664353363613763396434646565373333623638646463373639386237
-61616335363936303461326161396662383936336434626562613036633035373062663065356563
-39336233366262666631373439386632396637323737373165353538363335326664316531306261
-39333834383532393431343434396336393731303431336365636434646465613165386561393765
-33386333626366333262356539313134626533356465653239306235366562646533636334366137
-64323735633865343839653136343338313466346639323165663334343534376635656166633236
-37386365643737353361643935663039323537376562336238363937313036623765646538393530
-62353062333630613764633162363836643431373162303065653065666331303065616133323862
-65393731646566343939643031623835663064623561656533303962616335336234643939346165
-62613064373438366535353236346363613964326465333737306163633535653464636632336136
-36386230313931373836663062383164663463616535376431396632386431313161323463336562
-31323334326535613031303431376263356532316461343332376466646639646334353764376637
-34366334306166663030323134616234393665386533663964653538303430333135376234623139
-30353034373636396630396337643637363633306665636161613836343037303462336433643739
-37616635633139323866366138656336653065373437643861323966633639383965666133616236
-61313565383364333337643831313063626662633730343563666631643661623461333961363531
-32646562643130616132306262656536303330363431346562666361653435356138623435346334
-32313036633930323235613133643538393731353438373731306439646430316134313964303933
-63363638313266313532656232323939643031363162373837356230353934666131623436343464
-32643662366638643239363634653338343962326430313034343130323533316466613334393939
-66643131666330306530313635633734633565646336333366643330386262303737363866303037
-31396565303139656164306262666263656262383836313139343735346264616137636261386361
-37636466363865303362636661316436306439613262303933336135306264383262376439643663
-64663934323936353065623736623665643732346238336465656562373433656335313361323565
-37303562363466336139633065636236393236313831306330346234376639373635663638386330
-31653138396437313966353935633633336664333265303137343836653366333263656463623338
-66353061623431656366636137633035333138393135396433633566336435313337313835346130
-35633633323136653263623639613336623430333766666434333663616238306332613062626238
-38346438643638653063643133663962643831353866633434646436316239303030666338396263
-32393266333133616265646661633266393036343833316339633133336637623936306230363536
-33623264653939666466356666313465643863333765313930383761366335646237333735393230
-64353164666238313263373566376131323336636162396664653238306634303330616162333030
-37633561333932316233363666306234656335623033643962653932313266303564316135316334
-31633262646337393565366130656635626637336162366362383533303431396166303338353532
-64363366363961313633646231633165633836663939623039396662643462646431336230366439
-31633061366631353762663538316365616339653333356463336536323737356238383063313161
-34646233646137346362396134333665363230356339663664363538326530663539663436663562
-31303736333964366461386631326535333564336134316365336135326438363436326339333838
-65623039626261623563316537313231643065626136326163373835663536333666613734376337
-39373538656631326533616231653031346563613333663339383663316538663331643430326362
-36653931663732616532363765346537636333613666363137636363613331303639616263373736
-34386364663839613165323037363261656134323439613861623361333761386264303833346133
-39393031616665666133393532633731306662303531313137316266636137353363373835376336
-65323466356639356339326134626234353430623666393731386236366236343861633665616635
-31363761613566303031646535303737623731653834663465343766333939646130316533323536
-39373261363466623166666364366361356535646431643365353832333837333231306465333231
-36613561623535366565313538366232666537633064653838626334303438333662666231333935
-66623166633236393963316564346334663836623437353332346235623062393465326562313932
-34373139306162666163636130386162303538353537313365353133626262303233333064626331
-39386330343364653231363161323336303830303836626461666565333337666239333563636330
-62396463373663646238616536623033373732666130363963646566643963386162616433336631
-30623931666334383761313962626638636562626435333762356335333765373234643936353135
-66396464393161336461363166343639343534633962353130633436626133353736383637353436
-35383931316263373562663731333266393539383631373635633061393638643535643534313636
-33656335386431333337313834373135356538366131646539623963653039383034303563383938
-63363064646134383161326230363361626466393138633865393537663166306238383663383761
-34626639343564646430316133353466306364663464306263353231336566343234383464623566
-39383232633862373163353731653962363963666138656231643364663730386334633938323230
-62363335366539653437323865663535613232343832396138356335373331646435656336323264
-32386261633337323937353862636464346361303432633061653138316466316261363236646537
-65616261626264373464393362376436633865643264316139363866313838343133616230396633
-38643834643734383039653537313366663533646236343632636663616464326433653639646337
-34613161356335636232356436333533303435373161303862356331633132656162653937373936
-32326437323463336563353835313235333731663663303039656166343164356433376438663139
-33313762343737323566353938323836323832393331356530356633366437353265393365623261
-37343034663436303239
+35373831653035353532356664326238343363633038313666363937356535383930386266303436
+6332366333303937343433643364363130613535396162310a356266613334313536396464653165
+65653530383462356536656235396339663937643835313232313132663832373536376330336230
+6135303831393331620a653039326539656335336265383539366462346434633261373938356366
+33363566663565633966303634643565623136376462316638393263623163353433336132303032
+34376361373739643830396466666538323437306661633833616166393238653566616338396433
+39613164383262373236636464316334396264323833626362373938646363623461626637303261
+61623335353836336663343466613033366261326665633739346435323161653334333130633962
+66363436386130633338396662326234633334323461323239356431303434653563353234303236
+38663738656264653237626266636466373630636436336533663063613362633237376265356237
+34653934353532636561396462616361663532613530623766313961383361313635303938633362
+33306566663039373237396234346338613931626665656131356465323831663731303833643933
+66316435393664323166303334346239353030633466633561663431333530613835333337343966
+35663839653730643764323539333065633437333331353665366433386638353961663364633834
+65353166633039623837623939303763633561363931323561346165343437646134336665396264
+38306337613864343534353637376361323034653439663537316363643737316235623534396634
+30636435643464313834316133663764356133353666353036643032323263373439353363383465
+31343163653863633237633132646434663130356538313835663238383535396263616162346564
+61613132366136333436343434633139383536303061366338393637353764363164333761616635
+64316565636666333466313762373361326535383864666364663232643435323331353737316330
+31633738323533636262626264653632313638356239353235623636663638353334363934656264
+61666439623937363464316232343561623634323031326139356434363864323836313138303032
+33663965613961643434376333303332356139373066656163363737313833336634376533636130
+61353463366664356463363336633036386562646539386331633932613935643730636566313466
+31356263666631363635646466386233356133653834666133313136623165353238343731343161
+33326337663235643164663664326638656134663332376136383164336637333033306263396464
+65626431366564653832386462643431303137666532396635323966303734386365343632663733
+30313530323863393737623162646537383864353733633933613336323365363763313832623462
+66363563356462393938376632333338623232656130643666313463313765626431396132613165
+66663433303234383864323632303531616436663461396335333966303932666165353630316461
+39386438353965323036643433613232663935663938653030613662623136376365393666363535
+32333334333161656636343563663732626230353964626433633236396233613362393636343161
+63303863323961323466396532396132616437626365303164303832303834383466383437303966
+33656635623966326662656630363730326664643637383636616334393363636261663465336536
+65346438366165666331386531653835306461636431616539333366393239633536316166373862
+32613339366334343131636163376438646233393137626165383334306133653733646232306339
+66393435663132636235376130356132303037336535386461326365616137363332373262323634
+61303966376364623666366561306364356163353865613466636137323435313165663466366362
+31386462363739303430326438643436333635373236623237656230616631666131366462336561
+31623063393163623238386336336131353437373565333338643031363661366231383932626434
+66663836323666663865303938633338663563643130353334623831363961326435663765313634
+64356430383164636430653765653565653262336565353136633239393666316666633063323234
+66346537316432346531313362386533653765343561663338333734323437343935376239646464
+35336534366331393663663766616365666130353530306331336135373064373833343731643932
+32666362336236643237326636373739653032666165316164633138323233623936366462373964
+32323862633830623261623961316161323430396631633639396332393862363231313361326364
+30376331376238303934386562326266666264663434373564353839363634316664356531356632
+66376264336232346661633133373935633065363930636162326531303438366263346265393634
+61643833373630613836333337623931376231636533653536393563316436366665383030393637
+63373864376535663765313136363134656563303731666633653362366364316139313337633663
+62626139393430326136396331633532613135313762616133323665396531366634643932666266
+30663863653466643765663739666431663838376633316165353637663961376466376265363132
+61656363646534306165376134353265653230643039613964313462336161366636316433333930
+63393264396230363164373437306330346563346164633933393461363035643034346461336466
+38353336303264383166393633306663646265373932323663633032333162643831323331333262
+63313631653031633532626133303537616338306132396664643166333431663434653934383739
+39303738323966616262316164643935393261396361663066643239646638636431356164346635
+30653261656537613733313064643038396133363965656533666134343332663238626665626461
+32636162353337363762343832323234646564383565623065323136363536383766636539346339
+61643537653763383739613036363430316533363531383734633739323765613137623538636133
+31323731393461363765363231373061373831633433663036363539633230653430396335393037
+33646138366462323963653038356239656461323839636332313162333335313461636433653539
+34623161666333313038303934636539343334623730653762346334373533623264323532336433
+32616465353133336361323933393561356238643834646534376265353261316434386638373639
+39356264353833343635386536313961326662623435323236323835323266353131316462613031
+62326238373238633334646134303739386439613566336331623664313139303534303839626564
+36636237396666383966386163346631623534303733313531376432306466393330393061376132
+64303435653935303164336336653132653863366662616163326239383435306639333863616134
+61356164323966626161363864396533363130613434363365383262663937393439346665396134
+63383837643633323234363239626431363835666635366230353464373136336432366564636561
+65663236373664623039356165303365643439656561663236666337366535343731343163646636
+61373463393734313666313932613039336265343132613963373164663932623130333630336562
+32353662353439323338316335346261663831393861393637323734633939666534323532633437
+66626330626634666531346264373561353963636334333139383735666131373265313136396462
+36316266376338363965383731626130666237636638326637303330663535373461363265303239
+35383161326131353534313530623734346334383231373762303739636230613136303166356264
+64303937336461346162393365356436633239306434303163336161383236646265636331363333
+63626332643662613839323036383134346662393565656630656262663061656661663533333763
+63663934666361313961343338616532393661626365636134346331383861336431646662663033
+66316438646463636637656532376366636632326635666134366136656534366339306362383538
+32363763313265303035663436663132383539393166363961316631336464323363353466336161
+61303366306436383266616261316630396563373836663433663366616436613333323365653463
+65373730306234353331626634306162353930313539653738353434613461316665343961613338
+65613864643637363464616664663463303237313238373565393431653632366137366530643836
+61313264383238396430653631353431313562386561653362373830376261386538363666303039
+66316632633665393961346339313338363131373361653638633435383033393239383965636631
+32383865316662643636316362373065323364666262343139656363376265653638383134373837
+35383264633164326533633731306239333837353632326137383237333464356133353462613136
+30646436333935363263626131613864356163363633326662623465336362623061383166326163
+37356430663830366434653131653636626466353665666330343566306364303065393136383331
+34336239646563323664663733643130373463386135626561333361313039343932353530643061
+33323361383337623162363061343632643137623837386433366637356562346561373039666636
+65656534326632666532313333616236613032333536303664353564343962346163636132626130
+32373635323739353231336139643461363962333562316364666161643064356264353762383565
+64666663653761393135343238373064323064383263313330613265313230356133663964383336
+65633761643631656363323732646463623963616466396137383131363532393838376335303362
+31323538333264313163336362306461313538316330393261333061323162303330313536616662
+30373431343761313230616563303565326639353963326666313137633233303034333266613735
+63353835333739356231323538363336346337346236373661633132313331333865623238323436
+63373936356431316136376163383032653733313331666332383135313934336664633263613232
+30626339393365643735653761316636353632613663646539363238356332346230346638383432
+31346133346430306265353064643361396539613938326434366464303537643733613961313163
+33363564646230613865306335306262343735396330653864633039613565323033646436323334
+32393130613864646637616436653965356462636530653961653231383932626263336466613334
+37356132383361313762623637303337633763373363643938623230666230306238313135626434
+62663264373532656430393534643030386162656665323432323238626632363731613061646664
+33333033316664363365666164613333613439376462663565366132623833623432656537373737
+38646238653237633338363935626462383335333637356638396262313461336532646233353733
+39383038643039323062643165333039636233643034653930366133356135653839626136323336
+63646265366537373361386336396137383461336337323562646232653262396437366162383838
+37616330656562323238633362376637613964393630633037303039666663346634343133663831
+34383139656561393363323034343337643935353864343165626563646330656632646135333438
+34306533326364353561623762613162363431306364656634356138306233633565643938353432
+39616165316162383830626436623864316331356263626231376261356235666561323862323962
+38316237613065663233636133323566633335316361353838336466643966303530363163373431
+36613539323066643936353039353430643734303465343866303334366665623134373961346236
+32356137613036313066313864353766623961383738366262356364313061383735666436636430
+66353031323533333832643231343137653061623362306466386239396662383332343838613033
+36653263653038656562636666376663323363323239383630643663313539303461323434636139
+62626333313739336136623962343235636332623065626432326637313538346162663762656632
+61313332323338633938313164383536316161333263643037373863376634366538393065663936
+62623134373132363162386266656164653964353462303339376365316630333461363761643661
+65663064303666303236633630623230356432323931396138623862323863616535316165636463
+63373539396638373463393732653361313331666130636264616665636238623461616238613061
+32333539643961303566383365656165306264653536303365383639663738323666333030323631
+65363531346334386134633137393862316233633864363364376436663165363166366432373334
+64643338373539313131646138633963653033386236313663336266323664363661663535363539
+66373336626265636536313161386366623031633966393832633664313732643164623264373032
+37633666623534613137363765326136363462643735303130333532653433633831643736653763
+34656337656139633837343433663366303737316665613930383139646663373434316437346665
+63616130366232633835303439613638653362633764313333366435363734373536346230343031
+62623564646434396136303062653838616430306335393138316332626436353335393131303235
+33376664613931643536656238343366383332663737383331316339313330306364303861303634
+61356239356437636532623164656430356437396264333034656337366535656232353533623764
+65396639626635363032366232316561393365643737306135343134313032653963376564656265
+33356261643765336463366537643262323934383461613333633931333162313533613030386431
+32353263326537363034663138383163656461623462613738616463363830343138653138386532
+33626631623132313466633664623461643137643734326133353936653735306637343134366262
+35643864663737306230323331383637396165323864636533656331326165353264643061376661
+39313661333361616434333362643936663235363132353834373134303861656138346334613864
+32373163653362326537623335376236363962623561396430623439393038303838336166646234
+30323561333963306239623163373637666231656238623935313766393438646539326233656232
+63373136306635326263343333623461396333303339336561636132383564346163393263633464
+37313030653163643561373937663832613461383338383062636534303137656236323631613233
+62636639386236653737393631666635623832316361306165343035386536376261633636323332
+62383565663735613034323664363364653662393966323463613538393061336535393736663636
+39343939633932656633383030633164356130346138336263666235666632386235383162316161
+63393338313236353565393431623961636462336364313735623235343363373332626535393466
+35613730356164326662393362663735636430666337366633313430643631306462316133366636
+32393262326238656564663963613365613661343166383863623230643336616631656265343133
+33396435353933343963616263663337636264396136373163633865316333663839346665313036
+36353562346334623535653132313666396331383764366462623035333731336639346361333835
+36613435313839623233626633306562363461353036343639633362636537393362613432666232
+31623232653566636261623966356362333532303536353439326439643464623165626635333262
+38643236303534323238626635653332313966616566636331376438613633363432383330663037
+32376330336130653737623535333730643963356264653963313232326564303465323563326131
+38306164333061633033616561376263346635663237383133393961646330373039313265386661
+61326130653736643735646632343635343866346463383236343334653865643365633338303966
+63326536306166373334643662633434323034636538663866383736613736656437343439326237
+66653263613763383933376235636639613930616166313663393130323662636437396461393939
+37313431333237396465393466633835666365333439633266353634306431646564333262656232
+37363534373233646235303937333933346533306637333431396437633438396539313061313338
+35376137666132663566653939666532323232303466333536656364653337653337353063393939
+39343961643337353238383765396337653631396635323564333861366439393938623964363234
+34383262633838646363663738663461396238636661643466323464323732316135663165363236
+64633435393364323734356635323237306536383362613665353531383264386538353263396362
+31353862376166346538663231663732353066393736666266386231353130306232393864316137
+64643666303061653266343437313034633662323335383761386237336435633738646665633931
+36326532613466383365346138323466333432343166343037316663646562336661343739623738
+30326362366437323436376634316664633931646661323937333936666165303630666432353662
+32643931356664643266353530396436343730353766386663323735656132386363373063313936
+37383330623563363763343331623961333238633838616234616134373538636338336437353135
+35356538633239643233353162366530343739653030363365303134343631326265653664313462
+38333530303037303262323466626261313961386233656137376661656663303632666232336537
+36616266646534333238616434383564343938373132663839636165323061376163353536393361
+65653430366333623364343631333961333736633362623934653264323266666666653763363039
+34303966376662626635353839346163633936656366646236313239643239363066636237623832
+64633135356234623161303439373238303361326438656338626432323732316239656664336533
+31353666363131666537386563613062633861376230376339333836383538313666616338333263
+64663365333337626162353831363032326363373064303961303037383565663265386562383663
+34666637666162366337336637666238393561613665643161643564316539333038373839346132
+63303066323832653364623439313631386663653865646565383166336531306332303330666561
+63643933353332623564383933653964643463653930626434326562643230616432666638653665
+31386664643737336262643438353532346632343466646136613232393838363863643833663764
+38346530626339646462343662653863613765633863643564323866363333636431643734356161
+31663837346131646532633838393766336261613061326264333937393334356338383633383961
+37356539663132653166316362303064373431633036366465306263376239313336666361636531
+30633865383535643537333735323662393936613935373539306435653033616133366663623533
+61346630383433633634353431396530343032366535653466383162383134366234653961366634
+31373739333261656165376531616536623032636530303361333266643561333064306334393065
+65303639643137343933653538656336363932333465373432636161386232336339616363343539
+66616132343535323936616438313335313262363963393731336262333764303738316537386661
+36383161616462323431666464633464656561623638326163313132353534643565356635353130
+32653835306237663966376561623233636433613235343761336233643631313235346137316361
+65326338613066653930643065643465343762333661633266393232306563303739333539306463
+64646139623631613863376465616163643862333030373735376331336137613131346133393834
+32636563646664636533646262636532373562366463393633336564373833303337613666336432
+31613038646531663163336437373262396138363764383863633234643762616363643761613835
+30666630313937643833353935306165656435333039306531386365386437656238376339343634
+66626161643335393263336530646166623135623132353432323964376266643161313033626338
+39323832393562613164333538656236663139623633636461323833323236376238393062306263
+36343862623634323332643765663833396437353936393262376531366565393239643837393737
+61636461616166323431393933373537366665363630336635656533623731363132663337663733
+61393936386362373661323931396539393438373330396633326138303237366138613039393032
+33633234623034396231643266653564316138666139343062623361653530633065393736303064
+32643633623939326563363633346331633732616364333236653734626635646665336665393539
+61326363633666326631616436633236363863393364326130313632613064353230353065623965
+37366539346230353665653933653862386330313530353331396637353464633161663064366266
+61303265663237333065376665663138313737363432663839623032636338346533393763666531
+33343162373132623464653763343732653361663366363965353836363939306230656437333965
+31666463383664613966646132666431383665373061326361646463383262373061316636343734
+33633664383633643031383561646139656363653835666234616438626165613731396535333732
+38366136623437356532373436616237396135313732376635643138623537356138653738353537
+33656334616666353836633132303039346532323963363431383339666161623931623563623937
+39303134663565393930613939376564363332373561376534313537646265303239383138663336
+31316133326661303366646261326433363539663365363763643537353165336164333066636431
+33363738376465303435373064643336353962343732376461663638303566356435313935383830
+37653638336338306538383232626231666562333532356532383634313437626466376564303863
+39656333383835393035323966623834326232373136623963323239313134326335323338343661
+34313233323466383262656238623839616636396463616463366666656432643331333761613134
+62366531653639383265393833633132373361613833653239373665373537373231363361333062
+38356238646531383736363530616439363930363733383532303330386330353161643639363934
+66373436613633626139613033633432623766363633646632323632653264393435363336333731
+63366330346530326535663734323066613736383063616337616265396664633938633837636461
+33303633666361313930613866356233373335633066373939643437356537313861323534333735
+37656630653336313139616536363666303737643432316566633962633933656137633061336139
+36623636383237666139623931316535353631306665626131346361303164386663653534613530
+38366130343034313734643735373363643234373839623962653562623932323832386364616138
+63306265616132346330336139306262623435393739643131363639613235366662616266343338
+65323436383137313065303534376236663537376436383133343533663839316561313738376635
+61303965356334326138666165366338663632333366356465313765373165356530303038346565
+37383835636263306438373763393230323231326366333333353762333062373039316661323330
+34313564363766636439636465306133366161313033323331663765626338396138323930306365
+32636432653330383564633464396536333633363831336635386537316334323163656663653532
+36373430373363633333343134323136663361643934363265313464366630346338303535363233
+64353763316133666133656335633466633332353461326538653733666436383261616365363233
+36373139363533343130383734666161616439393539633032636338303636633837393338393366
+62373732303433663039353161356539646539613365616334643530663662363266316163353466
+31633563363966366636616135333831326262666135633464343334343564626538666236393131
+35353564616630653635373964656461376266323361313861663165373766353834313836303866
+61643532303562356135633837323136373432313131393065393531346131663264383966326562
+37643464326430313734653133313264336634363431633136336334333939366666633761386531
+38336664313031636661313837646461653133616139616265336530393531653237316165363062
+35643337393037653665336330666637656532366135613635393431306631393566376561306234
+37623265303633316434613938663366623966663434393161653635303033633862633533366364
+37343737393338323936333035333366613166653165363737343136383036386264613864376365
+30376261303665323439623866326633646337626534663031386164323362333230346134623466
+65323735646465346663643266643564633434316131353561303966653062616332306436343234
+63313131643436646237343231363331356632616234346362383431666566383263383463323734
+64666332363130353332643731623830616430636233366237613139623864646338303561653230
+39663039323939643939303130633333313565366463636232386235363435333534356135666463
+33666436383364376164396266303339393234396538313566316336393264656135613132643366
+63353037306137313436336631303436363665333633623366326538613834663337313462343530
+62333936396232666633303062623934653131616364363139656539386666663063623763643465
+35333035363738663731613637633337346361396363343334393637373434346564393766393633
+62633433346563396163356266366136353965306662316232666261393336626537393130643163
+35656333313864623334306437663537623831656536633232333433653336616464383561303937
+35376466333964396430346161306633393862366262623134333963656234633061356264333235
+34626438326536396663323338613762613536376234376535626134386237373963383865376238
+62396636653437396564303163613366353766333936303438393062646362373535306136316162
+34396136396636326632663033343538323530663764636166306130306164613936626139323939
+39643831613537616263323661336232336661643263383534616535366163323030626261346232
+30633265323734636330383037656638656134323438666339656164306634313364636232353263
+37623364376435363431

From 3450dfc09ff8c4bbc9d89c28ddc78b81f52e2ee6 Mon Sep 17 00:00:00 2001
From: nmdefries <42820733+nmdefries@users.noreply.github.com>
Date: Mon, 28 Oct 2024 08:21:11 -0400
Subject: [PATCH 13/13] Add `nchs-mortality` raw data backups and backup export
 utility (#2065)

* add helper fn in utils to save backup data to csv

* use helper to save nchs data to disk right after pulling

* add backup dir param

* import backup utility

* update arg name

* add gzip + fix old json template log + remove table_name

* fix current tests to take backup dirs and custom_run flag into account

* add logging

* fix log getsize of backup file

* lint

* lint

* lint

* lint

* add backup test

* remove deep copy

---------

Co-authored-by: minhkhul <minhkhul@andrew.cmu.edu>
Co-authored-by: minhkhul <118945681+minhkhul@users.noreply.github.com>
---
 _delphi_utils_python/delphi_utils/__init__.py |  13 +-
 _delphi_utils_python/delphi_utils/export.py   |  75 ++++++++++-
 .../nchs_mortality-params-prod.json.j2        |   1 +
 nchs_mortality/delphi_nchs_mortality/pull.py  |  22 +++-
 nchs_mortality/delphi_nchs_mortality/run.py   |   6 +-
 nchs_mortality/params.json.template           |   3 +-
 nchs_mortality/raw_data_backups/.gitignore    | 120 ++++++++++++++++++
 nchs_mortality/tests/conftest.py              |   4 +-
 .../tests/raw_data_backups/.gitignore         |   2 +
 nchs_mortality/tests/test_pull.py             |  18 ++-
 10 files changed, 245 insertions(+), 19 deletions(-)
 create mode 100644 nchs_mortality/raw_data_backups/.gitignore
 create mode 100644 nchs_mortality/tests/raw_data_backups/.gitignore

diff --git a/_delphi_utils_python/delphi_utils/__init__.py b/_delphi_utils_python/delphi_utils/__init__.py
index 7a418551d..ca5693eaf 100644
--- a/_delphi_utils_python/delphi_utils/__init__.py
+++ b/_delphi_utils_python/delphi_utils/__init__.py
@@ -4,15 +4,14 @@
 from __future__ import absolute_import
 
 from .archive import ArchiveDiffer, GitArchiveDiffer, S3ArchiveDiffer
-from .export import create_export_csv
-from .utils import read_params
-
-from .slack_notifier import SlackNotifier
-from .logger import get_structured_logger
+from .export import create_backup_csv, create_export_csv
 from .geomap import GeoMapper
-from .smooth import Smoother
-from .signal import add_prefix
+from .logger import get_structured_logger
 from .nancodes import Nans
+from .signal import add_prefix
+from .slack_notifier import SlackNotifier
+from .smooth import Smoother
+from .utils import read_params
 from .weekday import Weekday
 
 __version__ = "0.3.25"
diff --git a/_delphi_utils_python/delphi_utils/export.py b/_delphi_utils_python/delphi_utils/export.py
index 8ac5de48e..82493032e 100644
--- a/_delphi_utils_python/delphi_utils/export.py
+++ b/_delphi_utils_python/delphi_utils/export.py
@@ -1,16 +1,18 @@
 """Export data in the format expected by the Delphi API."""
 # -*- coding: utf-8 -*-
+import gzip
+import logging
 from datetime import datetime
-from os.path import join
+from os.path import getsize, join
 from typing import Optional
-import logging
 
-from epiweeks import Week
 import numpy as np
 import pandas as pd
+from epiweeks import Week
 
 from .nancodes import Nans
 
+
 def filter_contradicting_missing_codes(df, sensor, metric, date, logger=None):
     """Find values with contradictory missingness codes, filter them, and log."""
     columns = ["val", "se", "sample_size"]
@@ -132,3 +134,70 @@ def create_export_csv(
             export_df = export_df.sort_values(by="geo_id")
         export_df.to_csv(export_file, index=False, na_rep="NA")
     return dates
+
+
+def create_backup_csv(
+    df: pd.DataFrame,
+    backup_dir: str,
+    custom_run: bool,
+    issue: Optional[str] = None,
+    geo_res: Optional[str] = None,
+    sensor: Optional[str] = None,
+    metric: Optional[str] = None,
+    logger: Optional[logging.Logger] = None,
+):
+    """Save data for use as a backup.
+
+    This function is meant to save raw data fetched from data sources.
+    Therefore, it avoids manipulating the data as much as possible to
+    preserve the input.
+
+    When only required arguments are passed, data will be saved to a file of
+    the format `<export_dir>/<today's date as YYYYMMDD>.csv`. Optional arguments
+    should be passed if the source data is fetched from different tables or
+    in batches by signal, geo, etc.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+        Columns: geo_id, timestamp, val, se, sample_size
+    backup_dir: str
+        Backup directory
+    custom_run: bool
+        Flag indicating if the current run is a patch, or other run where
+        backups aren't needed. If so, don't save any data to disk
+    issue: Optional[str]
+        The date the data was fetched, in YYYYMMDD format. Defaults to "today"
+        if not provided
+    geo_res: Optional[str]
+        Geographic resolution of the data
+    sensor: Optional[str]
+        Sensor that has been calculated (cumulative_counts vs new_counts)
+    metric: Optional[str]
+        Metric we are considering, if any.
+    logger: Optional[logging.Logger]
+        Pass a logger object here to log information about name and size of the backup file.
+
+    Returns
+    ---------
+    dates: pd.Series[datetime]
+        Series of dates for which CSV files were exported.
+    """
+    if not custom_run:
+        # Label the file with today's date (the date the data was fetched).
+        if not issue:
+            issue = datetime.today().strftime("%Y%m%d")
+
+        backup_filename = [issue, geo_res, metric, sensor]
+        backup_filename = "_".join(filter(None, backup_filename)) + ".csv.gz"
+        backup_file = join(backup_dir, backup_filename)
+
+        with gzip.open(backup_file, "wt", newline="") as f:
+            df.to_csv(f, index=False, na_rep="NA")
+
+        if logger:
+            logger.info(
+                "Backup file created",
+                backup_file=backup_file,
+                backup_size=getsize(backup_file),
+            )
diff --git a/ansible/templates/nchs_mortality-params-prod.json.j2 b/ansible/templates/nchs_mortality-params-prod.json.j2
index dbd39598b..4b0d0c4f7 100644
--- a/ansible/templates/nchs_mortality-params-prod.json.j2
+++ b/ansible/templates/nchs_mortality-params-prod.json.j2
@@ -1,6 +1,7 @@
 {
   "common": {
     "daily_export_dir": "./daily_receiving",
+    "backup_dir": "./raw_data_backups",
     "log_filename": "/var/log/indicators/nchs_mortality.log",
     "weekly_export_dir": "/common/covidcast/receiving/nchs-mortality"
   },
diff --git a/nchs_mortality/delphi_nchs_mortality/pull.py b/nchs_mortality/delphi_nchs_mortality/pull.py
index 18bbfd59a..ad54e457a 100644
--- a/nchs_mortality/delphi_nchs_mortality/pull.py
+++ b/nchs_mortality/delphi_nchs_mortality/pull.py
@@ -1,15 +1,17 @@
 # -*- coding: utf-8 -*-
 """Functions for pulling NCHS mortality data API."""
 
+import logging
 from typing import Optional
 
 import numpy as np
 import pandas as pd
+from delphi_utils import create_backup_csv
+from delphi_utils.geomap import GeoMapper
 from sodapy import Socrata
 
-from delphi_utils.geomap import GeoMapper
+from .constants import METRICS, NEWLINE, RENAME
 
-from .constants import METRICS, RENAME, NEWLINE
 
 def standardize_columns(df):
     """Rename columns to comply with a standard set.
@@ -22,7 +24,13 @@ def standardize_columns(df):
     return df.rename(columns=dict(rename_pairs))
 
 
-def pull_nchs_mortality_data(socrata_token: str, test_file: Optional[str] = None):
+def pull_nchs_mortality_data(
+    socrata_token: str,
+    backup_dir: str,
+    custom_run: bool,
+    logger: Optional[logging.Logger] = None,
+    test_file: Optional[str] = None,
+):
     """Pull the latest NCHS Mortality data, and conforms it into a dataset.
 
     The output dataset has:
@@ -40,6 +48,10 @@ def pull_nchs_mortality_data(socrata_token: str, test_file: Optional[str] = None
     ----------
     socrata_token: str
         My App Token for pulling the NCHS mortality data
+    backup_dir: str
+        Directory to which to save raw backup data
+    custom_run: bool
+        Flag indicating if the current run is a patch. If so, don't save any data to disk
     test_file: Optional[str]
         When not null, name of file from which to read test data
 
@@ -60,6 +72,10 @@ def pull_nchs_mortality_data(socrata_token: str, test_file: Optional[str] = None
         client = Socrata("data.cdc.gov", socrata_token)
         results = client.get("r8kw-7aab", limit=10**10)
         df = pd.DataFrame.from_records(results)
+
+    create_backup_csv(df, backup_dir, custom_run=custom_run, logger=logger)
+
+    if not test_file:
         # drop "By Total" rows
         df = df[df["group"].transform(str.lower) == "by week"]
 
diff --git a/nchs_mortality/delphi_nchs_mortality/run.py b/nchs_mortality/delphi_nchs_mortality/run.py
index 50ce46cfb..4e88e9d61 100644
--- a/nchs_mortality/delphi_nchs_mortality/run.py
+++ b/nchs_mortality/delphi_nchs_mortality/run.py
@@ -59,6 +59,8 @@ def run_module(params: Dict[str, Any]):
                 days=date.today().weekday() + 2)
         export_start_date = export_start_date.strftime('%Y-%m-%d')
     daily_export_dir = params["common"]["daily_export_dir"]
+    backup_dir = params["common"]["backup_dir"]
+    custom_run = params["common"].get("custom_run", False)
     socrata_token = params["indicator"]["socrata_token"]
     test_file = params["indicator"].get("test_file", None)
 
@@ -70,7 +72,9 @@ def run_module(params: Dict[str, Any]):
         daily_arch_diff.update_cache()
 
     stats = []
-    df_pull = pull_nchs_mortality_data(socrata_token, test_file)
+    df_pull = pull_nchs_mortality_data(
+        socrata_token, backup_dir, custom_run=custom_run, test_file=test_file, logger=logger
+    )
     for metric in METRICS:
         for geo in ["state", "nation"]:
             if metric == 'percent_of_expected_deaths':
diff --git a/nchs_mortality/params.json.template b/nchs_mortality/params.json.template
index ed16c620c..2e829de24 100644
--- a/nchs_mortality/params.json.template
+++ b/nchs_mortality/params.json.template
@@ -2,7 +2,8 @@
   "common": {
     "daily_export_dir": "./daily_receiving",
     "weekly_export_dir": "./receiving",
-    "log_filename": "/var/log/indicators/nchs_mortality.log",
+    "backup_dir": "./raw_data_backups",
+    "log_filename": "./nchs_mortality.log",
     "log_exceptions": false
   },
   "indicator": {
diff --git a/nchs_mortality/raw_data_backups/.gitignore b/nchs_mortality/raw_data_backups/.gitignore
new file mode 100644
index 000000000..552154e09
--- /dev/null
+++ b/nchs_mortality/raw_data_backups/.gitignore
@@ -0,0 +1,120 @@
+# You should hard commit a prototype for this file, but we
+# want to avoid accidental adding of API tokens and other
+# private data parameters
+params.json
+
+# Do not commit output files
+receiving/*.csv
+
+# Remove macOS files
+.DS_Store
+
+# virtual environment
+dview/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+coverage.xml
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+.static_storage/
+.media/
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
diff --git a/nchs_mortality/tests/conftest.py b/nchs_mortality/tests/conftest.py
index 6ad0f9c59..383d1c782 100644
--- a/nchs_mortality/tests/conftest.py
+++ b/nchs_mortality/tests/conftest.py
@@ -14,8 +14,10 @@
 
 PARAMS = {
   "common": {
+    "custom_run": True,
     "daily_export_dir": "./daily_receiving",
-    "weekly_export_dir": "./receiving"
+    "weekly_export_dir": "./receiving",
+    "backup_dir": "./raw_data_backups"
   },
   "indicator": {
     "export_start_date": "2020-04-11",
diff --git a/nchs_mortality/tests/raw_data_backups/.gitignore b/nchs_mortality/tests/raw_data_backups/.gitignore
new file mode 100644
index 000000000..2b7efbb36
--- /dev/null
+++ b/nchs_mortality/tests/raw_data_backups/.gitignore
@@ -0,0 +1,2 @@
+*.csv
+*.gz
\ No newline at end of file
diff --git a/nchs_mortality/tests/test_pull.py b/nchs_mortality/tests/test_pull.py
index fa58b04a5..4f18210f6 100644
--- a/nchs_mortality/tests/test_pull.py
+++ b/nchs_mortality/tests/test_pull.py
@@ -1,3 +1,4 @@
+import os
 import pytest
 
 import pandas as pd
@@ -34,7 +35,7 @@ def test_standardize_columns(self):
         pd.testing.assert_frame_equal(expected, df)
 
     def test_good_file(self):
-        df = pull_nchs_mortality_data(SOCRATA_TOKEN, "test_data.csv")
+        df = pull_nchs_mortality_data(SOCRATA_TOKEN, backup_dir = "", custom_run = True, test_file = "test_data.csv")
 
         # Test columns
         assert (
@@ -90,9 +91,20 @@ def test_good_file(self):
     def test_bad_file_with_inconsistent_time_col(self):
         with pytest.raises(ValueError):
             pull_nchs_mortality_data(
-                SOCRATA_TOKEN, "bad_data_with_inconsistent_time_col.csv"
+                SOCRATA_TOKEN, backup_dir = "", custom_run = True, test_file = "bad_data_with_inconsistent_time_col.csv"
             )
 
     def test_bad_file_with_missing_cols(self):
         with pytest.raises(ValueError):
-            pull_nchs_mortality_data(SOCRATA_TOKEN, "bad_data_with_missing_cols.csv")
+            pull_nchs_mortality_data(SOCRATA_TOKEN, backup_dir = "", custom_run = True, test_file = "bad_data_with_missing_cols.csv")
+
+    def test_backup_today_data(self):
+        today = pd.Timestamp.today().strftime("%Y%m%d")
+        backup_dir = "./raw_data_backups"
+        pull_nchs_mortality_data(SOCRATA_TOKEN, backup_dir = backup_dir, custom_run = False, test_file = "test_data.csv")
+        backup_file = f"{backup_dir}/{today}.csv.gz"
+        backup_df = pd.read_csv(backup_file)
+        source_df = pd.read_csv("test_data/test_data.csv")
+        pd.testing.assert_frame_equal(source_df, backup_df)
+        if os.path.exists(backup_file):
+            os.remove(backup_file)