diff --git a/.bumpversion.cfg b/.bumpversion.cfg index de364861c..547bad12d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.55 +current_version = 0.3.56 commit = True message = chore: bump covidcast-indicators to {new_version} tag = False diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 0dcfa85c2..66b0afc62 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -12,7 +12,9 @@ The production branch is configured to automatically deploy to our production en * everything else -All other branches are development branches. We don't enforce a naming policy. +All other branches are development branches. We don't enforce a naming policy, but it is recommended to prefix all branches you create with your name, username, or initials (e.g. `username/branch-name`). + +We don't forbid force-pushing, but please keep to a minimum and be careful of using when modifying a branch at the same time as others. ## Issues @@ -29,7 +31,7 @@ So, how does one go about developing a pipeline for a new data source? ### tl;dr 1. Create your new indicator branch from `main`. -2. Build it using the appropriate template, following the guidelines in the included README.md and REVIEW.md files. +2. Build it using the [indicator template](https://github.com/cmu-delphi/covidcast-indicators/tree/main/_template_python), following the guidelines in the included README.md, REVIEW.md, and INDICATOR_DEV_GUIDE.md files. 3. Make some stuff! 4. When your stuff works, push your development branch to remote, and open a PR against `main` for review. 5. Once your PR has been merged, consult with a platform engineer for the remaining production setup needs. They will create a deployment workflow for your indicator including any necessary production parameters. Production secrets are encrypted in the Ansible vault. This workflow will be tested in staging by admins, who will consult you about any problems they encounter. @@ -50,7 +52,7 @@ git checkout -b dev-my-feature-branch ### Creating your indicator -Create a directory for your new indicator by making a copy of `_template_r` or `_template_python` depending on the programming language you intend to use. If using Python, add the name of the directory to the list found in `jobs > build > strategy > matrix > packages` in `.github/workflows/python-ci.yml`, which will enable automated checks for your indicator when you make PRs. The template copies of `README.md` and `REVIEW.md` include the minimum requirements for code structure, documentation, linting, testing, and method of configuration. Beyond that, we don't have any established restrictions on implementation; you can look at other existing indicators see some examples of code layout, organization, and general approach. +Create a directory for your new indicator by making a copy of `_template_python`. (We also make a `_template_r` available, but R should be only used as a last resort, due to complications using it in production.) Add the name of the directory to the list found in `jobs > build > strategy > matrix > packages` in `.github/workflows/python-ci.yml`, which will enable automated checks for your indicator when you make PRs. The template copies of `README.md` and `REVIEW.md` include the minimum requirements for code structure, documentation, linting, testing, and method of configuration. Beyond that, we don't have any established restrictions on implementation; you can look at other existing indicators see some examples of code layout, organization, and general approach. * Consult your peers with questions! :handshake: @@ -62,7 +64,7 @@ Once you have something that runs locally and passes tests you set up your remot git push -u origin dev-my-feature-branch ``` -You can then draft public API documentation for people who would fetch this +You can then draft [public API documentation](https://cmu-delphi.github.io/delphi-epidata/) for people who would fetch this data from the API. Public API documentation is kept in the delphi-epidata repository, and there is a [template Markdown file](https://github.com/cmu-delphi/delphi-epidata/blob/main/docs/api/covidcast-signals/_source-template.md) @@ -104,7 +106,8 @@ We use a branch-based git workflow coupled with [Jenkins](https://www.jenkins.io * Package - Tar and gzip the built environment. * Deploy - Trigger an Ansible playbook to place the built package onto the runtime host, place any necessary production configuration, and adjust the runtime envirnemnt (if necessary). -There are several additional Jenkins-specific files that will need to be created for each indicator, as well as some configuration additions to the runtime host. It will be important to pair with a platform engineer to prepare the necessary production environment needs, test the workflow, validate on production, and ultimately sign off on a production release. +There are several additional Jenkins-specific files that will need to be created for each indicator, as well as some configuration additions to the runtime host. +It will be important to pair with a platform engineer to prepare the necessary production environment needs, test the workflow, validate on production, and ultimately sign off on a production release. ### Preparing container images of indicators diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index a9f1ccae9..e7cd8548b 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -86,7 +86,7 @@ jobs: - name: Release run: | make release - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: delphi_utils path: _delphi_utils_python/dist/*.tar.gz diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 3e1ee9689..6e23c61d2 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -51,7 +51,7 @@ jobs: with: python-version: 3.8 cache: "pip" - cache-dependency-path: "setup.py" + cache-dependency-path: "pyproject.toml" - name: Install testing dependencies run: | python -m pip install --upgrade pip diff --git a/Jenkinsfile b/Jenkinsfile index 1b9485ca5..3fcdc904a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,6 +18,10 @@ def deploy_production = [:] pipeline { agent any + environment { + // Set the PATH variable to include the pyenv shims directory. + PATH = "/var/lib/jenkins/.pyenv/shims:${env.PATH}" + } stages { stage('Build dev/feature branch') { when { diff --git a/_delphi_utils_python/.bumpversion.cfg b/_delphi_utils_python/.bumpversion.cfg index 722a91e30..08040f0ec 100644 --- a/_delphi_utils_python/.bumpversion.cfg +++ b/_delphi_utils_python/.bumpversion.cfg @@ -1,9 +1,9 @@ [bumpversion] -current_version = 0.3.24 +current_version = 0.3.25 commit = True message = chore: bump delphi_utils to {new_version} tag = False -[bumpversion:file:setup.py] +[bumpversion:file:pyproject.toml] [bumpversion:file:delphi_utils/__init__.py] diff --git a/_delphi_utils_python/DEVELOP.md b/_delphi_utils_python/DEVELOP.md index 2407e29a8..53ffde93f 100644 --- a/_delphi_utils_python/DEVELOP.md +++ b/_delphi_utils_python/DEVELOP.md @@ -9,7 +9,7 @@ To install the module in your default version of Python, run the following from this directory: ``` -pip install . +pip install -e '.[dev]' ``` As described in each of the indicator code directories, you will want to install diff --git a/_delphi_utils_python/Makefile b/_delphi_utils_python/Makefile index 79d7f7943..60f7d2af0 100644 --- a/_delphi_utils_python/Makefile +++ b/_delphi_utils_python/Makefile @@ -6,12 +6,12 @@ venv: install: venv . env/bin/activate; \ pip install wheel ; \ - pip install -e . + pip install -e '.[dev]' install-ci: venv . env/bin/activate; \ - pip install wheel ; \ - pip install . + pip install 'build[virtualenv]' pylint pytest pydocstyle wheel twine ; \ + pip install '.[dev]' lint: . env/bin/activate; pylint delphi_utils --rcfile=../pyproject.toml @@ -30,4 +30,5 @@ clean: release: lint test . env/bin/activate ; \ - python setup.py sdist bdist_wheel + pip install 'build[virtualenv]' ; \ + python -m build --sdist --wheel \ No newline at end of file diff --git a/_delphi_utils_python/README.md b/_delphi_utils_python/README.md index 0ac6350bf..e1b091cae 100644 --- a/_delphi_utils_python/README.md +++ b/_delphi_utils_python/README.md @@ -22,13 +22,22 @@ Source code can be found here: ## Logger Usage +To make our structured logging as useful as it can be, particularly within the context of how we use logs in Elastic, the `event` argument (typically the first unnamed arg) should be a static string (to make filtering easier), and each dynamic/varying value should be specified in an individual meaningfully- and consistently-named argument to the logger call (for use in filtering, thresholding, grouping, visualization, etc). + +### Commonly used argument names: +- data_source +- geo_type +- signal +- issue_date +- filename + Single-thread usage. ```py from delphi_utils.logger import get_structured_logger logger = get_structured_logger('my_logger') -logger.info('Hello, world!') +logger.info('Hello', name='World') ``` Multi-thread usage. diff --git a/_delphi_utils_python/delphi_utils/__init__.py b/_delphi_utils_python/delphi_utils/__init__.py index 7ff828440..ca5693eaf 100644 --- a/_delphi_utils_python/delphi_utils/__init__.py +++ b/_delphi_utils_python/delphi_utils/__init__.py @@ -4,15 +4,14 @@ from __future__ import absolute_import from .archive import ArchiveDiffer, GitArchiveDiffer, S3ArchiveDiffer -from .export import create_export_csv -from .utils import read_params - -from .slack_notifier import SlackNotifier -from .logger import get_structured_logger +from .export import create_backup_csv, create_export_csv from .geomap import GeoMapper -from .smooth import Smoother -from .signal import add_prefix +from .logger import get_structured_logger from .nancodes import Nans +from .signal import add_prefix +from .slack_notifier import SlackNotifier +from .smooth import Smoother +from .utils import read_params from .weekday import Weekday -__version__ = "0.3.24" +__version__ = "0.3.25" diff --git a/_delphi_utils_python/delphi_utils/export.py b/_delphi_utils_python/delphi_utils/export.py index 37f8faf98..82493032e 100644 --- a/_delphi_utils_python/delphi_utils/export.py +++ b/_delphi_utils_python/delphi_utils/export.py @@ -1,16 +1,18 @@ """Export data in the format expected by the Delphi API.""" # -*- coding: utf-8 -*- +import gzip +import logging from datetime import datetime -from os.path import join +from os.path import getsize, join from typing import Optional -import logging -from epiweeks import Week import numpy as np import pandas as pd +from epiweeks import Week from .nancodes import Nans + def filter_contradicting_missing_codes(df, sensor, metric, date, logger=None): """Find values with contradictory missingness codes, filter them, and log.""" columns = ["val", "se", "sample_size"] @@ -22,8 +24,10 @@ def filter_contradicting_missing_codes(df, sensor, metric, date, logger=None): for mask in masks: if not logger is None and df.loc[mask].size > 0: logger.info( - "Filtering contradictory missing code in " + - "{0}_{1}_{2}.".format(sensor, metric, date.strftime(format="%Y-%m-%d")) + "Filtering contradictory missing code", + sensor=sensor, + metric=metric, + date=date.strftime(format="%Y-%m-%d"), ) df = df.loc[~mask] elif logger is None and df.loc[mask].size > 0: @@ -130,3 +134,70 @@ def create_export_csv( export_df = export_df.sort_values(by="geo_id") export_df.to_csv(export_file, index=False, na_rep="NA") return dates + + +def create_backup_csv( + df: pd.DataFrame, + backup_dir: str, + custom_run: bool, + issue: Optional[str] = None, + geo_res: Optional[str] = None, + sensor: Optional[str] = None, + metric: Optional[str] = None, + logger: Optional[logging.Logger] = None, +): + """Save data for use as a backup. + + This function is meant to save raw data fetched from data sources. + Therefore, it avoids manipulating the data as much as possible to + preserve the input. + + When only required arguments are passed, data will be saved to a file of + the format `/.csv`. Optional arguments + should be passed if the source data is fetched from different tables or + in batches by signal, geo, etc. + + Parameters + ---------- + df: pd.DataFrame + Columns: geo_id, timestamp, val, se, sample_size + backup_dir: str + Backup directory + custom_run: bool + Flag indicating if the current run is a patch, or other run where + backups aren't needed. If so, don't save any data to disk + issue: Optional[str] + The date the data was fetched, in YYYYMMDD format. Defaults to "today" + if not provided + geo_res: Optional[str] + Geographic resolution of the data + sensor: Optional[str] + Sensor that has been calculated (cumulative_counts vs new_counts) + metric: Optional[str] + Metric we are considering, if any. + logger: Optional[logging.Logger] + Pass a logger object here to log information about name and size of the backup file. + + Returns + --------- + dates: pd.Series[datetime] + Series of dates for which CSV files were exported. + """ + if not custom_run: + # Label the file with today's date (the date the data was fetched). + if not issue: + issue = datetime.today().strftime("%Y%m%d") + + backup_filename = [issue, geo_res, metric, sensor] + backup_filename = "_".join(filter(None, backup_filename)) + ".csv.gz" + backup_file = join(backup_dir, backup_filename) + + with gzip.open(backup_file, "wt", newline="") as f: + df.to_csv(f, index=False, na_rep="NA") + + if logger: + logger.info( + "Backup file created", + backup_file=backup_file, + backup_size=getsize(backup_file), + ) diff --git a/_delphi_utils_python/delphi_utils/flash_eval/eval_day.py b/_delphi_utils_python/delphi_utils/flash_eval/eval_day.py index 660fca042..3c8012803 100644 --- a/_delphi_utils_python/delphi_utils/flash_eval/eval_day.py +++ b/_delphi_utils_python/delphi_utils/flash_eval/eval_day.py @@ -153,8 +153,7 @@ def output(evd_ranking, day, lag, signal, logger): p_text += f"\t{start_link}|*{index}*, {'{:.2f}'.format(value)}>\n" else: break - name = f"Signal: {signal} Lag: {lag}" - logger.info(name, payload=p_text) + logger.info("FLaSH: worth inspecting", signal=signal, lag=lag, payload=p_text) def evd_ranking_fn(ts_streams, EVD_max, EVD_min): diff --git a/_delphi_utils_python/delphi_utils/geomap.py b/_delphi_utils_python/delphi_utils/geomap.py index be6df2d24..f6df4a919 100644 --- a/_delphi_utils_python/delphi_utils/geomap.py +++ b/_delphi_utils_python/delphi_utils/geomap.py @@ -443,7 +443,7 @@ def add_population_column( --------- data: pd.DataFrame The dataframe with a FIPS code column. - geocode_type: {"fips", "zip"} + geocode_type: The type of the geocode contained in geocode_col. geocode_col: str, default None The name of the column containing the geocodes. If None, uses the geocode_type @@ -671,8 +671,10 @@ def aggregate_by_weighted_sum( to a from_geo, e.g. "wastewater collection site"). to_geo: str The column name of the geocode to aggregate to. - sensor: str + sensor_col: str The column name of the sensor to aggregate. + time_col: str + The column name of the timestamp to aggregate over. population_column: str The column name of the population to weight the sensor by. diff --git a/_delphi_utils_python/delphi_utils/logger.py b/_delphi_utils_python/delphi_utils/logger.py index c0e4502a8..30fd78059 100644 --- a/_delphi_utils_python/delphi_utils/logger.py +++ b/_delphi_utils_python/delphi_utils/logger.py @@ -1,11 +1,11 @@ """Structured logger utility for creating JSON logs. -See the delphi_utils README.md for usage examples. +To make our structured logging as useful as it can be, particularly within the context of how we use logs in Elastic, +the `event` argument (typically the first unnamed arg) should be a static string (to make filtering easier), +and each dynamic/varying value should be specified in an individual meaningfully- and consistently-named argument +to the logger call (for use in filtering, thresholding, grouping, visualization, etc) -The Delphi group uses two ~identical versions of this file. -Try to keep them in sync with edits, for sanity. - https://github.com/cmu-delphi/covidcast-indicators/blob/main/_delphi_utils_python/delphi_utils/logger.py - https://github.com/cmu-delphi/delphi-epidata/blob/dev/src/common/logger.py +See the delphi_utils README.md for usage examples. """ import contextlib diff --git a/_delphi_utils_python/delphi_utils/runner.py b/_delphi_utils_python/delphi_utils/runner.py index abc28ba19..9083371aa 100644 --- a/_delphi_utils_python/delphi_utils/runner.py +++ b/_delphi_utils_python/delphi_utils/runner.py @@ -51,6 +51,7 @@ def run_indicator_pipeline(indicator_fn: Callable[[Params], None], #Get version and indicator name for startup ind_name = indicator_fn.__module__.replace(".run", "") + #Check for version.cfg in indicator directory if os.path.exists("version.cfg"): with open("version.cfg") as ver_file: @@ -59,9 +60,15 @@ def run_indicator_pipeline(indicator_fn: Callable[[Params], None], if "current_version" in line: current_version = str.strip(line) current_version = current_version.replace("current_version = ", "") - #Logging - Starting Indicator - logger.info(f"Started {ind_name} with covidcast-indicators version {current_version}") - else: logger.info(f"Started {ind_name} without version.cfg") + logger.info( + "Started a covidcast-indicator", + indicator_name=ind_name, + current_version=current_version, + ) + else: + logger.info( + "Started a covidcast-indicator without version.cfg", indicator_name=ind_name + ) indicator_fn(params) validator = validator_fn(params) @@ -77,8 +84,10 @@ def run_indicator_pipeline(indicator_fn: Callable[[Params], None], break time.sleep(1) else: - logger.error(f"Flash step timed out ({timer} s), terminating", - elapsed_time_in_seconds = round(time.time() - start, 2)) + logger.error( + "Flash step timed out, terminating", + elapsed_time_in_seconds=round(time.time() - start, 2), + ) t1.terminate() t1.join() if validator: diff --git a/_delphi_utils_python/pyproject.toml b/_delphi_utils_python/pyproject.toml new file mode 100644 index 000000000..c47590a29 --- /dev/null +++ b/_delphi_utils_python/pyproject.toml @@ -0,0 +1,57 @@ +[build-system] +requires = ["setuptools", "setuptools-scm>=8.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "delphi-utils" +version = "0.3.25" +description = "Shared Utility Functions for Indicators" +readme = "README.md" +requires-python = "== 3.8.*" +license = { text = "MIT License" } +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3.8", + "License :: OSI Approved :: MIT License", +] +dependencies = [ + "boto3", + "covidcast", + "cvxpy", + "epiweeks", + "gitpython", + "importlib_resources>=1.3", + "numpy", + "pandas>=1.1.0", + "requests", + "slackclient", + "scs<3.2.6", # TODO: remove this ; it is a cvxpy dependency, and the excluded version appears to break our jenkins build. see: https://github.com/cvxgrp/scs/issues/283 + "structlog", + "xlrd", # needed by Pandas to read Excel files +] + +[project.urls] +Homepage = "https://github.com/cmu-delphi/covidcast-indicators" + +[project.optional-dependencies] +dev = [ + "darker[isort]~=2.1.1", + "pylint==2.8.3", + "pytest", + "pydocstyle", + "pytest-cov", + "mock", + "moto~=4.2.14", + "requests-mock", + "freezegun", +] +flash = ["scipy"] + +[tool.setuptools.packages.find] +where = ["."] +include = ["delphi_utils"] +namespaces = true + +[tool.setuptools.package-data] +"delphi_utils.data" = ["20*/*.csv"] diff --git a/_delphi_utils_python/setup.py b/_delphi_utils_python/setup.py deleted file mode 100644 index 3dee89b53..000000000 --- a/_delphi_utils_python/setup.py +++ /dev/null @@ -1,48 +0,0 @@ -from setuptools import setup -from setuptools import find_packages - -with open("README.md", "r") as f: - long_description = f.read() - -required = [ - "boto3", - "covidcast", - "cvxpy", - "scs<3.2.6", # TODO: remove this ; it is a cvxpy dependency, and the excluded version appears to break our jenkins build. see: https://github.com/cvxgrp/scs/issues/283 - "darker[isort]~=2.1.1", - "epiweeks", - "freezegun", - "gitpython", - "importlib_resources>=1.3", - "mock", - "moto~=4.2.14", - "numpy", - "pandas>=1.1.0", - "pydocstyle", - "pylint==2.8.3", - "pytest-cov", - "pytest", - "requests-mock", - "slackclient", - "structlog", - "xlrd" -] - -setup( - name="delphi_utils", - version="0.3.24", - description="Shared Utility Functions for Indicators", - long_description=long_description, - long_description_content_type="text/markdown", - author="", - author_email="", - url="https://github.com/cmu-delphi/", - install_requires=required, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Programming Language :: Python :: 3.8", - ], - packages=find_packages(), - package_data={'': ['data/20*/*.csv']} -) diff --git a/_delphi_utils_python/tests/test_export.py b/_delphi_utils_python/tests/test_export.py index 3e72f1d7f..c9c1f8483 100644 --- a/_delphi_utils_python/tests/test_export.py +++ b/_delphi_utils_python/tests/test_export.py @@ -323,12 +323,13 @@ def test_export_df_with_missingness(self, tmp_path): @mock.patch("delphi_utils.logger") def test_export_df_with_contradictory_missingness(self, mock_logger, tmp_path): - + sensor = "test" + geo_res = "state" create_export_csv( df=self.DF3.copy(), export_dir=tmp_path, - geo_res="state", - sensor="test", + sensor=sensor, + geo_res=geo_res, logger=mock_logger ) assert set(listdir(tmp_path)) == set( @@ -339,8 +340,9 @@ def test_export_df_with_contradictory_missingness(self, mock_logger, tmp_path): ] ) assert pd.read_csv(join(tmp_path, "20200315_state_test.csv")).size > 0 + date_str = datetime.strftime(self.TIMES[0], "%Y-%m-%d") mock_logger.info.assert_called_once_with( - "Filtering contradictory missing code in test_None_2020-02-15." + "Filtering contradictory missing code", sensor=sensor, metric=None, date=date_str ) def test_export_sort(self, tmp_path): diff --git a/_template_python/INDICATOR_DEV_GUIDE.md b/_template_python/INDICATOR_DEV_GUIDE.md index 2b8b6333c..f58a1ada9 100644 --- a/_template_python/INDICATOR_DEV_GUIDE.md +++ b/_template_python/INDICATOR_DEV_GUIDE.md @@ -50,6 +50,7 @@ This is the general extract-transform-load procedure used by all COVIDcast indic 7. Deliver the CSV output files to the `receiving/` directory on the API server. Adding a new indicator typically means implementing steps 1-3. Step 4 is included via the function ` create_export_csv`. Steps 5 (the validator), 6 (the archive differ) and 7 (acquisition) are all handled by runners in production. + ## Step 0: Keep revision history (important!) If the data provider doesn’t provide or it is unclear if they provide historical versions of the data, immediately set up a script (bash, Python, etc) to automatically (e.g. cron) download the data every day and save locally with versioning. @@ -195,7 +196,14 @@ Generally, indicators have: Do other geo handling (e.g. finding and reporting DC as a state). * `constants.py`: Lists of geos to produce, signals to produce, dataset ids, data source URL, etc. -Your code should be _extensively_ commented! Especially note sections where you took an unusual approach (make sure to say why and consider briefly discussing alternate approaches). +Your code should be _extensively_ commented! Especially note sections where you took an unusual approach (make sure to say why and consider briefly discussing alternate approaches that were discarded or could be useful in the future). + +#### Development environment + +Make sure you have a functional environment with python 3.8.15+. +For local runs, the makefile’s make install target will set up a local virtual environment with necessary packages. + +(If working in R (very much NOT recommended), local runs can be run without a virtual environment or using the [`renv` package](https://rstudio.github.io/renv/articles/renv.html), but production runs should be set up to use Docker.) #### Function stubs @@ -235,12 +243,22 @@ def api_call(token: str): After that, generalize your code to be able to be run on all geos of interest, take settings from params.json, use constants for easy maintenance, with extensive documentation, etc. -#### Development environment +#### Testing -Make sure you have a functional environment with python 3.8.15+. -For local runs, the makefile’s make install target will set up a local virtual environment with necessary packages. +As a general rule, it helps to decompose your functions into operations for which you can write unit tests. +To run the tests, use `make test` in the top-level indicator directory. -(If working in R (very much NOT recommended), local runs can be run without a virtual environment or using the [`renv` package](https://rstudio.github.io/renv/articles/renv.html), but production runs should be set up to use Docker.) +Unit tests are required for all functions. +Integration tests are highly desired, but may be difficult to set up depending on where the data is being fetched from. +Mocking functions are useful in this case. + +#### Dealing with dates + +We keep track of two different date fields for each dataset. The first field is called "reference value" (field name `time_value`) and tracks the date that a value is reported _for_, that is, when the event happened. The second field is called "issue date" or "version" (field name `issue`) and tracks when a value was recorded, not when it happened. + +For example, flu test positivity of 80% for a reference date of Jan 1 and an issue date of Jan 5 means that _on_ Jan 1, the test positivity rate was 80%. But we only received and recorded the value on Jan 5, 4 days later (AKA a lag of 4 days). + +It's important to track issue date because many data sources are revised over time, and reported values can change substantially between issues. #### Dealing with data-types @@ -255,14 +273,16 @@ E.g. which geo values are allowed, should every valid date be present in some wa In an ideal case, the data exists at one of our [already covered geos](https://cmu-delphi.github.io/delphi-epidata/api/covidcast_geography.html): -* State: state_code (string, leftpadded to 2 digits with 0) or state_id (string) +* Zip code * FIPS (state+county codes, string leftpadded to 5 digits with 0) -* ZIP * MSA (metro statistical area, int) * HRR (hospital referral region, int) +* State: state_code (string, leftpadded to 2 digits with 0) or state_id (string) +* HHS ([Department of Health and Human Services-defined regions](https://www.hhs.gov/about/agencies/iea/regional-offices/index.html)) +* Nation -If you want to map from one of these to another, the [`delphi_utils.geomapper`](https://github.com/cmu-delphi/covidcast-indicators/blob/6912077acba97e835aff7d0cd3d64309a1a9241d/_delphi_utils_python/delphi_utils/geomap.py) utility covers most cases. -A brief example of aggregating from states to hhs regions via their population: +If you want to map from one of these to a higher level, the [`delphi_utils.geomapper`](https://github.com/cmu-delphi/covidcast-indicators/blob/6912077acba97e835aff7d0cd3d64309a1a9241d/_delphi_utils_python/delphi_utils/geomap.py) utility covers most cases. +Here's a brief example of aggregating from states to hhs regions via their population: ```{python} from delphi_utils.geomap import GeoMapper @@ -274,19 +294,6 @@ hhs_version = geo_mapper.replace_geocode(df, "state_code","hhs", new_col = "geo_ This example is taken from [`hhs_hosp`](https://github.com/cmu-delphi/covidcast-indicators/blob/main/hhs_hosp/delphi_hhs/run.py); more documentation can be found in the `geomapper` class definition. -#### Implement a Missing Value code system - -The column is described [here](https://cmu-delphi.github.io/delphi-epidata/api/missing_codes.html). - -#### Local testing - -As a general rule, it helps to decompose your functions into operations for which you can write unit tests. -To run the tests, use `make test` in the top-level indicator directory. - -Unit tests are required for all functions. -Integration tests are highly desired, but may be difficult to set up depending on where the data is being fetched from. -Mocking functions are useful in this case. - #### Naming Indicator and signal names need to be approved by [@RoniRos](https://www.github.com/RoniRos). @@ -324,6 +331,52 @@ Using this tag dictionary, we can interpret the following signals as * `confirmed_admissions_influenza_1d_prop` = raw (unsmoothed) daily ("1d") confirmed influenza hospital admissions ("confirmed_admissions_influenza") per 100,000 population ("prop"). * `confirmed_admissions_influenza_1d_prop_7dav` = the same as above, but smoothed with a 7-day moving average ("7dav"). +#### Implement a Missing Value code system + +The column is described [here](https://cmu-delphi.github.io/delphi-epidata/api/missing_codes.html). + +#### Implement a patching method + +After normal data reporting is restored following an outage, we would like to be able to easily reconstruct the version history of the data. To do so, implement a `patch` method that runs an indicator's main `run_module` for every issue date in a range. An [example patch module](https://github.com/cmu-delphi/covidcast-indicators/blob/b784f30/google_symptoms/delphi_google_symptoms/patch.py). + +An outage can be external to Delphi, e.g. the data provider was unable to provide new data on the historically-expected schedule, or internal, e.g. Delphi code had a bug that caused our pipeline to fail. The goal of the patch feature is to recreate every missing issue _as if we had ingested it on the correct day_. + +The patch feature should be easy to use. The only manual parts should be modifying `params.json`, and running the patch module and acquisition. Any setup that needs to be done (e.g. cache creation, dir creation) should be done automatically as part of the patch function. + +All patch modules should expect settings from `params.json` of the form + +``` +{ + "common": { + ... + "custom_run": true + }, + "validation": { + ... + }, + "patch": { + "patch_dir": "/", + "start_issue": "2024-04-20", + "end_issue": "2024-04-21" + } +} +``` + +The `custom_run` parameter should [default to false](https://github.com/cmu-delphi/covidcast-indicators/blob/d435bf0f0d5880ddf8905ea60f242976e6702342/nssp/delphi_nssp/run.py#L73), and [warn](https://github.com/cmu-delphi/covidcast-indicators/blob/d435bf0f0d5880ddf8905ea60f242976e6702342/nssp/delphi_nssp/run.py#L75-L83) if parameters and arguments disagree. + +Patching should generate data for that range of issue dates, and store them in batch issue format: +`/issue_//xxx.csv`. + +Acquisition in `delphi-epidata` includes [code that allow files in this issue-specific structure](https://github.com/cmu-delphi/delphi-epidata/blob/694d89ad763fa85bd644e1f64552c9bc85f688ef/src/acquisition/covidcast/csv_to_database.py#L43C32-L43C61) to be added to the database. This output format is designed to match the `issue`-type acquisition format. The issue-specific mode is triggered with the flag `specific_issue_date`. [A Cronicle job](https://cronicle-prod-01.delphi.cmu.edu/#Schedule?sub=edit_event&id=elh59ynwobf) has already been set up to call acquisition using the flag; please use it to load patches into the database. + +Sometimes source data is already versioned, and to reconstruct an issue we simply need to filter the source data to include only values that would have been available on that issue day. If we receive data drops directly, we can filter by the file creation date instead. + +However, it is not always possible to reconstruct issues; many datasets aren't versioned by the provider. If a source has no revisions (for example, `google-symptoms`), then we can guess which dates of data would have been available that issue day based on the normal lag of the source. For example, `google-symptoms` normally has a lag of 4 days, i.e. "today" the most recent data we see in the source data is from 4 days ago. So to reconstruct data for issue 2024-01-10, we just need to report data with a `time_value` (reference date) from 2024-01-06 and earlier. (How much earlier depends on the behavior we normally expect from the indicator code; if we normally report 2 weeks of data, filter to 2024-01-06 - 14 days through 2024-01-06.) + +Some datasets, such as those on healthdata.gov, provide metadata indicating when certain rows were updated. + +In other cases (such as datasetes that both have revisions _and_ don't track revisions), please discuss with the indicator stakeholder and consider [what you know about how the data works](#step-1-exploratory-analysis). + ### Statistical review The data produced by the new indicator needs to be sanity-checked. @@ -419,14 +472,17 @@ Refer to [this guide](https://docs.google.com/document/d/1Bbuvtoxowt7x2_8USx_JY- * Add module name to the `build` job in `.github/workflows/python-ci.yml`. This allows github actions to run on this indicator code, which includes unit tests and linting. -* Add top-level directory name to `indicator_list` in `Jenkinsfile`. +* Add module name to the ["Copy version to indicator directory" step](https://github.com/cmu-delphi/covidcast-indicators/blob/f01185767a9847d8082baf4f1e17be50a39047c2/.github/workflows/create-release.yml#L64) in `.github/workflows/create-release.yml`. +* Add top-level directory name to [`indicator_list` in `Jenkinsfile`](https://github.com/cmu-delphi/covidcast-indicators/blob/f01185767a9847d8082baf4f1e17be50a39047c2/Jenkinsfile#L13). This allows your code to be automatically deployed to staging after your branch is merged to main, and deployed to prod after `covidcast-indicators` is released. * Create `ansible/templates/{top_level_directory_name}-params-prod.json.j2` based on your `params.json.template` with some adjustment: * "export_dir": "/common/covidcast/receiving/{data-source-name}" * "log_filename": "/var/log/indicators/{top_level_directory_name}.log" +* Define any sensitive variables as "secrets" in the [Ansible `vars.yaml`](https://github.com/cmu-delphi/covidcast-indicators/blob/main/ansible/vars.yaml) and [vault](https://github.com/cmu-delphi/covidcast-indicators/blob/main/ansible/vault.yaml). +Refer to [this guide](https://docs.google.com/document/d/1Bbuvtoxowt7x2_8USx_JY-yTo-Av3oAFlhyG-vXGG-c/edit#heading=h.8kkoy8sx3t7f) for more vault info. +* Add configs for Sir Complains-a-Lot ("sirCAL") alerting in sirCAL's [local](https://github.com/cmu-delphi/covidcast-indicators/blob/main/sir_complainsalot/params.json.template) and [Ansible](https://github.com/cmu-delphi/covidcast-indicators/blob/main/ansible/templates/sir_complainsalot-params-prod.json.j2) params templates. Pay attention to the receiving/export directory, as well as how you can store credentials in vault. -Refer to [this guide](https://docs.google.com/document/d/1Bbuvtoxowt7x2_8USx_JY-yTo-Av3oAFlhyG-vXGG-c/edit#heading=h.8kkoy8sx3t7f) for more vault info. ### Staging diff --git a/_template_python/pyproject.toml b/_template_python/pyproject.toml new file mode 100644 index 000000000..2f84aaa00 --- /dev/null +++ b/_template_python/pyproject.toml @@ -0,0 +1,42 @@ +[build-system] +requires = ["setuptools", "setuptools-scm>=8.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "delphi_NAME" +version = "0.1.0" +description = "SHORT DESCRIPTION" +readme = "README.md" +requires-python = "== 3.8.*" +license = { text = "MIT License" } +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3.8", + "License :: OSI Approved :: MIT License", +] +dependencies = [ + "delphi-utils", + "numpy", + "pandas>=1.1.0", +] + +[project.urls] +Homepage = "https://github.com/cmu-delphi/covidcast-indicators" + +[project.optional-dependencies] +dev = [ + "darker[isort]~=2.1.1", + "pylint==2.8.3", + "pytest", + "pydocstyle", + "pytest-cov", + "mock", + "moto~=4.2.14", + "requests-mock", + "freezegun", +] + +[tool.setuptools.packages.find] +where = ["."] +namespaces = true diff --git a/_template_python/setup.py b/_template_python/setup.py deleted file mode 100644 index d7bc44078..000000000 --- a/_template_python/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -from setuptools import setup -from setuptools import find_packages - -required = [ - "covidcast", - "darker[isort]~=2.1.1", - "delphi-utils", - "numpy", - "pandas", - "pydocstyle", - "pylint==2.8.3", - "pytest-cov", - "pytest", -] - -setup( - name="delphi_NAME", - version="0.1.0", - description="SHORT DESCRIPTION", - author="", - author_email="", - url="https://github.com/cmu-delphi/covidcast-indicators", - install_requires=required, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Programming Language :: Python :: 3.8", - ], - packages=find_packages(), -) diff --git a/ansible/templates/nchs_mortality-params-prod.json.j2 b/ansible/templates/nchs_mortality-params-prod.json.j2 index dbd39598b..4b0d0c4f7 100644 --- a/ansible/templates/nchs_mortality-params-prod.json.j2 +++ b/ansible/templates/nchs_mortality-params-prod.json.j2 @@ -1,6 +1,7 @@ { "common": { "daily_export_dir": "./daily_receiving", + "backup_dir": "./raw_data_backups", "log_filename": "/var/log/indicators/nchs_mortality.log", "weekly_export_dir": "/common/covidcast/receiving/nchs-mortality" }, diff --git a/ansible/vault.yaml b/ansible/vault.yaml index bf979841a..1a674592b 100644 --- a/ansible/vault.yaml +++ b/ansible/vault.yaml @@ -1,265 +1,265 @@ $ANSIBLE_VAULT;1.1;AES256 -63313737356234663364633839353638393064663937643934333430643832363065646138333038 -3133663766613235613137383736383030636236323166370a366235353833306336336432376333 -61336662656336336530326463663761366232393063326365366561386636636165396430623165 -6131643963386231650a356137646637333266613966343631656136303438303932326638363038 -30653364343436383433663639393633356633393836613263663537366335316130323365653736 -31353831343864383932393661383431346337353863643063636632313666613562306237623930 -63633061343263626139613337376333653562653432363763306535663735373433336636376437 -37646265373831613461363838303635306662633637643535623961376564653265663633656537 -66663134383638636138386137626133613964363333643133356365356237313935363531373066 -61626337396232316535376339306162316266366261383630623466306634643533326166386234 -65323262363337343261633638336364333230343035653032383031333431623032643432353136 -66653865623931313266326161343064383938636435306636383033313731323762383462303238 -34643562306264316130316266623838613361393462663361386234626661353838363036323937 -31653432623236303263643739326561306131316337313030636563363838353638353665343339 -38383938333164383337623634363430366232666231323539373239363266623339306434313162 -66393238663031376231386431666132643139383731323535313839643864633239346364303333 -61623238303038663833396663633963323863636139616631373036376337396262643862633939 -61646662653238396532613962376237623561313461333061356663343664306563386539376633 -32616433333163643066313330666263653939303932653835303562623061363833636435643534 -62336561306335346664333833306335323364613039653263373334616134313962613630386262 -63306630333739616137346338383964346365663836353166653236636663366539396431323032 -33366362346462303865623937363032363233316530656534383164393138323565306139306461 -35656266633636393631353435386337333039303535343233353134333839376163316262316533 -62313764306330316339623033636436313539326664353339326635613462316362333661363635 -66326565636439353466656663303663636237333965393961656637623933636366653562663564 -38373463626139313234623333306261666239303135333965656537393338323763666131366330 -38303435353765366162373430393231656139316238613839353432306231343130373038376135 -33396232636131653534666164313131333861336130636664643730393631626361353739303835 -31666132643830376335343938376338666538663865613132313130323736616237323134333031 -35656663346464383233646362663736633230613437643834383833633637656161303865343362 -64303636616262383865343332353139633432626565666233656436653761626563653639613334 -64313030643363323665613965353565393762323065323735306337373833373035396430376137 -62383063626430313532653261616230336164666461613966623032323965623962373830626236 -37356361326562613634376261336436346535333831643764343932356334616535343364316563 -64333731383831323165663530613736323461643836363438343134633635336431376664366364 -37326565613631613333313632666538383966363739666338626431643734636565616665653938 -37623839313030653733626263656666363362373135623562653462363330366136373938663532 -30343436303863636636353931656562303565626631323238356463373330393161373932373965 -34636638633764616430336131363734323932656135343032636564383231363731663963666663 -38376263333938643933363336373637396237303961653062343866363632663532336434643063 -63353739343033396130643638396365336332626439313231396361383736373036393838623532 -35383566333238626131343530313463326631313462333462396233613438333365613932663237 -35376232383238323636343335643236656135333766386565326337643466363064616131616430 -33643131363639363164316438656364386532626134386666613533643831326162363961666332 -62303938666565393938393133363734613365303136623638613532333033346465633830333164 -30333363376338373063373338393435626566356338393036303939353236633339373538613661 -63653064616338313135373238383432363138356665386638343434326337343862313431333535 -33633761333836376238376530636236613165353266363433663065633038626532343462353164 -61396530373532343831303331316236373834363538333632323664633363636133373438376234 -34343136363734333963383865646361653233393132373938633930366462383632333839353861 -63353030373137633161396165636366613337646461353265616263303663393636303231386234 -39356263323831313439333630376365343530336539663139373933336265303834623734656339 -35326239333163346335316138303831303063613839353164663563363737303834343132393237 -30376132303862386137626333653539303133366233303335356636303737353661306163313165 -35623865336130633366633131343838343830613537626539306566306439393737343438386162 -65393937323337306439663435346132386530353564333730626533373537646461646339323530 -31613033376539653363636332346238316632306463343436346239633364306638663464396639 -30653661616636663762353461366330613337643531336239346163306234313330663435336439 -61333334393937303933393965663030656232336464313135323164323030656134663030346366 -63613638623666363936613065643137636662306662396332643362333439393732326263623461 -37396135353033353832393763356439643566373738336438623234396365613136616432633636 -31363735393664303039333233313636326564363963616365646337373065363937306164643239 -38636131633563616339343734313836613462646636323163336136623034613965633433633035 -33396636323662343638616463303731306665616232643161613937356530303562383033626365 -64393835636365383432653330633538336435373336623833626337363264373665383530663538 -62316266326634376133666334336563656662666536353434306232373465623835346233316664 -63643966613363366363363265626435626132346532333864623834323061383235623034316139 -31386163623133366365353766373764323833363035333835383163656363386338386433323365 -38616437353530643739306365386134376364623665313965656533323334306438393031623434 -62373632633636663832353465353666333362663266623633376337393330353635386533343235 -63633938343062336436356334616633393331666562313165666664343133393935633465393739 -66633031323531346630343265663933666338623765663163336435343632303132313139656637 -64653030633938326439336331393763316161326464303231353066626464633836323431303666 -63316230623539333663623961393434393239373132363935326635383930626436636135386663 -61343433666365336638393639323038333032363462663734353031376463383531393661396333 -32386666316230613463333964323332373765333962383764613931643266313839333639383964 -34663761336166366532383138623230613037646165363039613138316464306135366366373136 -32353937663863333738356166623836303863343665353564326562313937333965646132626664 -62323434313964636564393432636534396261346265353336636564386338363137386632306661 -32653938393831646136613464663532313531636331633264303230306134623431333564666130 -66363330326661393339373235383530656562316531373666656638343038336164323363343637 -61653131313462356538656232643237623038666664663435363134363162306538383930373637 -63633539623265393137393332336463653132363661356365656237663461613764663837323336 -34613732363466353836363064383661626431363061613430616562396161366430363362336338 -35626265336562336134633538326365383365383766663731633738323430633365663266393061 -63396365303462386665343666346133303661616435663566303634363833303663383464623237 -30393364326333393932303462386435616163366165646463383765396132663166316666656462 -36346266636539663336646231366463353839363639613765313663303232633163356565643565 -61633163373335316362343835663364626336656236373361373830303463363965333432363134 -37333961636163373963613263353039636266376265333632363163363334353563323533663064 -33323234623035336232326238623838383533616533626665643133636233313062343137376263 -36653762666562633837356362383662663338616635646535393139356336346639326264643835 -61356531316366373038666166643434656633623635333238303831656338336539313931383762 -63376434373435646562623934346437316465303565313564613365663533303665363662653039 -61343363383438633236643664396533373134633963386638396236386561373161376365303832 -33333563306662623137633231613566353361643735666364653233666136313230633338323764 -65373334633836623666396637363166323631383835356666336461636433653464366138343630 -33323064353661633131303830613534636164643432303165376437346161373739333633616533 -63356331373061663463623435653632353433316261343562303837336135616636353864313432 -34643362303063363861646639336236623533636463306137393034633837643331653739643662 -65613763373665343830363063643738326439306164633533376339366265336466313432643136 -36386561343862376463306438326539393163356165313936323939323831343339393433643135 -38303862393932306161383236663466653933353932616537383965316464643762363331336137 -35663533316466613862373634323535663434303563386232306339613239633732346238323163 -61313330356631396135313765646435336633643038653835363436386363373138666266656162 -39363833396165323236376635626130623034336139336233333931323933306664333533653361 -35646139666131643430353731326137363135316236383833373631396266666463626464353365 -65666630633437343235336561366566366139393965383663623534643333383637646662313134 -30363161613464656632313536373631633166313162303761303835343338643366306330343232 -65656238346132383134626564333336306463643962383530343064343263643435363638366637 -62393337313732353230366536383434386332383837303161366166663430656335333733653738 -34386637353132323939323162323734643936666563663037353832666330366538333138653536 -62343462353835333232376666653538346564343566303135333434303963393032623564303034 -62653732666135383666623061646339366563316564613031303639323732666362326630643931 -35333436313236666563376131336233396336626161313863393962386638656634346137646530 -66376563336432633331643939623834353162613030326163383565396334383839643661363737 -36666363636237366537376565656465663834646665333930653132393461636334303334396161 -30363332303938336462646632303435643635363163333431343032616432646233353463396632 -32303938623263366634376536353764616361393430393361316263336239333633396666666566 -62633032383131636162333336306139356239313732383939323662386333363635663634616633 -61393262616162623966393264323466343738303566653862353939303637303039363136623662 -66336166326336373962306138373137313731656330373036656662363964646665386131656334 -36353432636238636161633131363161316438306430356233383232303163633764633736343264 -62346664663362656634666634353939663937613863383037333861386135353338613461663466 -36386463373938383637653733333637656362663061396466373563653861386630346330626261 -33656164363362306433356137346335363830306461303431333461393238373838336235666463 -33316235363338383435303939656331636639623733636566396536383461353534613331356139 -36333165303562313236623265613136653935346237343361636136343835363162656636313564 -63373063336434376232393662316634343734333534363537373637636264643666383835303536 -65653534393839623063396433663630363263626232613065663031333235636333643062313065 -30333330636339313032653431646639373465633633386166646139343461643964356263623738 -34356661663339386135383732643039613034326436643334656564646564326333323239333235 -38383562353263623265623230323333623536323139366462316665626537656132356561616361 -39313639333638656565393531646564643038336564623635343438376533343732663635653031 -30633231383431663834313066396362636238386332633665663661633733373836656335313239 -33386638393935346361376637663539663832393763343662643363623165316331336463663134 -62633832336236633831653837643764323031663734376262373866643036613931306633343065 -34383637646534373135306230376163356461646136393463623866303834333636633536336263 -33373066366333663063623638643035343161353633633631643461383332353263303962303432 -33303737336137643463343333633065643237333539383563376662373833623931333366373833 -65653733653363613561316365646363353535323838386133646532353364306639646561323937 -66396663383934616665653833646661333630303737666239383365363530393430323439353030 -36393064393163663234393665393233393534376236633139626662666330666461633437646466 -35346262663265386236366431623937396330616230306533616266616265653638643732653164 -39336232376239656134356363636332343631653133653231366338373535313363646430653137 -34653435396338306436346664383932336130393966643561373062396438363839363938313036 -38303761616630303265313665623535633032626333323932386535376166353534663262653130 -39313664616166383530343463633561306137313162346335356562346636383239323765343962 -35393462653034386662306438363533333865343634343565653164613566313331653832613035 -31646462363465316133653433653165633530643666633031643133656638323132653335626431 -37613631366332663165306264373736353532376431326131623764353238623762393763373365 -33663738626263663936373138613764386437656432303135656265636365353834616465633062 -66306435396639663533633465396664303936313936613330646564613630303131373437313538 -31306637643434303635316563323662643433643531626332613437353261313963373933623062 -37303331366561386233353665346236643363326537616532306236646533646163343862386333 -33343335373864636232323538356663343230323362346533323865383630613666366364383964 -65633434353763323033386231623737363033376438616162333339643463323730333435663566 -31323139326465393261653335366330313936383939313735353133393938313066636533336664 -31346166613434326665333866633762663338636335373965613435396539363834326338386564 -37626666653961376265343138323934373234636161373037656530643832663764346632623734 -38306461646231636666623166323334323765373639343665663537356564356332366632623434 -62353666306635393237313838313739313330623466376638653766613734663964666334313834 -32353137333564343334613937353766663532323063663233623865656439323864393838643038 -30613164356138316534643166633834663531653664306563333834636632346635666662323435 -62366662353930373637613032316639323836626166663766396461626130393031383331343734 -61633165663461353336303761613637323435383033363437396531613332323037346562643835 -38643938616436303062323763396338353365653738323838353865313533666639646462633933 -33323235666334623137363331313565366631633533643834323530333462386539626335303935 -37323161613762333932303465656238333435333162383338396565313665373537336439336539 -63326133353762346366313963383636666438633439346436306661366563373934663034653037 -32656463363338656562613332613566323531353964613139376530303437396139656530396166 -61626561663836643333653736316530393566666364633563633336666431616531663233343065 -39353036336566613934316434306261626363636536643265393230623630316562636265373766 -39396166373565616464386234363639323839346234303063666239323337366330663538303532 -32366338323263323530633762336438303037623330316666383836356630346235333065383263 -35333734623461373331393436666164656433646361323638353430373739656338306438333231 -31623632356231616663613734343965326662636330646533363538303562323433643430333565 -61356639383464393538666134396634366264333338396435363336656334376530343162373464 -66396336363136666335613737666330646536663364643733353761306534363863656461383331 -37373061333138373331356535316338626238326134626330356234376133316163663264643333 -38646131323063343032613765663232323164303338303634336366353238663532356335376638 -64323165663261363431646532383037666262626638373131653663303931383535636236316531 -36343736363431316661383031626633343334653164393231303339303564373665623434326639 -64383062373566313562653836326665623531316531303334306431383466313763636338383035 -30383762336666343639623138623961366365303631656438636466613663653537623635636262 -32616662303765343237373936623664383035333437663436333231353239663966343764303761 -30663837356435383333333237393664386631313337366437333735626136313561336465303934 -61623934346330626435303034366636306364376466623364366433326539653466353232616465 -61393632336463383262663235323665353931613065326565656336396261356235303737383662 -62333035316265646533653061363266303037653766366262326631376466343932313530653238 -30323165633338386637333865313761646633633735393238643464653638343934376435343962 -34323934383038323439626239336332653330366263343961356438373939373065393036633536 -64643839636261633234666663303063386438303132633166636631356166316535666132356161 -36333535663235663562383935663463336638383565326630393337663930376632666364353239 -61336233393166623937386666313464626633353430663064366631323831333739356261393837 -35333631383034313463336263353135656234623535303233353465316166623132333063386366 -35393234646139323132326163396632353763353262363139613562346266656266626435336330 -30616566353163326336386266666136643233393662643562376337366433306139393438373130 -37383565643931646632323664353363613763396434646565373333623638646463373639386237 -61616335363936303461326161396662383936336434626562613036633035373062663065356563 -39336233366262666631373439386632396637323737373165353538363335326664316531306261 -39333834383532393431343434396336393731303431336365636434646465613165386561393765 -33386333626366333262356539313134626533356465653239306235366562646533636334366137 -64323735633865343839653136343338313466346639323165663334343534376635656166633236 -37386365643737353361643935663039323537376562336238363937313036623765646538393530 -62353062333630613764633162363836643431373162303065653065666331303065616133323862 -65393731646566343939643031623835663064623561656533303962616335336234643939346165 -62613064373438366535353236346363613964326465333737306163633535653464636632336136 -36386230313931373836663062383164663463616535376431396632386431313161323463336562 -31323334326535613031303431376263356532316461343332376466646639646334353764376637 -34366334306166663030323134616234393665386533663964653538303430333135376234623139 -30353034373636396630396337643637363633306665636161613836343037303462336433643739 -37616635633139323866366138656336653065373437643861323966633639383965666133616236 -61313565383364333337643831313063626662633730343563666631643661623461333961363531 -32646562643130616132306262656536303330363431346562666361653435356138623435346334 -32313036633930323235613133643538393731353438373731306439646430316134313964303933 -63363638313266313532656232323939643031363162373837356230353934666131623436343464 -32643662366638643239363634653338343962326430313034343130323533316466613334393939 -66643131666330306530313635633734633565646336333366643330386262303737363866303037 -31396565303139656164306262666263656262383836313139343735346264616137636261386361 -37636466363865303362636661316436306439613262303933336135306264383262376439643663 -64663934323936353065623736623665643732346238336465656562373433656335313361323565 -37303562363466336139633065636236393236313831306330346234376639373635663638386330 -31653138396437313966353935633633336664333265303137343836653366333263656463623338 -66353061623431656366636137633035333138393135396433633566336435313337313835346130 -35633633323136653263623639613336623430333766666434333663616238306332613062626238 -38346438643638653063643133663962643831353866633434646436316239303030666338396263 -32393266333133616265646661633266393036343833316339633133336637623936306230363536 -33623264653939666466356666313465643863333765313930383761366335646237333735393230 -64353164666238313263373566376131323336636162396664653238306634303330616162333030 -37633561333932316233363666306234656335623033643962653932313266303564316135316334 -31633262646337393565366130656635626637336162366362383533303431396166303338353532 -64363366363961313633646231633165633836663939623039396662643462646431336230366439 -31633061366631353762663538316365616339653333356463336536323737356238383063313161 -34646233646137346362396134333665363230356339663664363538326530663539663436663562 -31303736333964366461386631326535333564336134316365336135326438363436326339333838 -65623039626261623563316537313231643065626136326163373835663536333666613734376337 -39373538656631326533616231653031346563613333663339383663316538663331643430326362 -36653931663732616532363765346537636333613666363137636363613331303639616263373736 -34386364663839613165323037363261656134323439613861623361333761386264303833346133 -39393031616665666133393532633731306662303531313137316266636137353363373835376336 -65323466356639356339326134626234353430623666393731386236366236343861633665616635 -31363761613566303031646535303737623731653834663465343766333939646130316533323536 -39373261363466623166666364366361356535646431643365353832333837333231306465333231 -36613561623535366565313538366232666537633064653838626334303438333662666231333935 -66623166633236393963316564346334663836623437353332346235623062393465326562313932 -34373139306162666163636130386162303538353537313365353133626262303233333064626331 -39386330343364653231363161323336303830303836626461666565333337666239333563636330 -62396463373663646238616536623033373732666130363963646566643963386162616433336631 -30623931666334383761313962626638636562626435333762356335333765373234643936353135 -66396464393161336461363166343639343534633962353130633436626133353736383637353436 -35383931316263373562663731333266393539383631373635633061393638643535643534313636 -33656335386431333337313834373135356538366131646539623963653039383034303563383938 -63363064646134383161326230363361626466393138633865393537663166306238383663383761 -34626639343564646430316133353466306364663464306263353231336566343234383464623566 -39383232633862373163353731653962363963666138656231643364663730386334633938323230 -62363335366539653437323865663535613232343832396138356335373331646435656336323264 -32386261633337323937353862636464346361303432633061653138316466316261363236646537 -65616261626264373464393362376436633865643264316139363866313838343133616230396633 -38643834643734383039653537313366663533646236343632636663616464326433653639646337 -34613161356335636232356436333533303435373161303862356331633132656162653937373936 -32326437323463336563353835313235333731663663303039656166343164356433376438663139 -33313762343737323566353938323836323832393331356530356633366437353265393365623261 -37343034663436303239 +35373831653035353532356664326238343363633038313666363937356535383930386266303436 +6332366333303937343433643364363130613535396162310a356266613334313536396464653165 +65653530383462356536656235396339663937643835313232313132663832373536376330336230 +6135303831393331620a653039326539656335336265383539366462346434633261373938356366 +33363566663565633966303634643565623136376462316638393263623163353433336132303032 +34376361373739643830396466666538323437306661633833616166393238653566616338396433 +39613164383262373236636464316334396264323833626362373938646363623461626637303261 +61623335353836336663343466613033366261326665633739346435323161653334333130633962 +66363436386130633338396662326234633334323461323239356431303434653563353234303236 +38663738656264653237626266636466373630636436336533663063613362633237376265356237 +34653934353532636561396462616361663532613530623766313961383361313635303938633362 +33306566663039373237396234346338613931626665656131356465323831663731303833643933 +66316435393664323166303334346239353030633466633561663431333530613835333337343966 +35663839653730643764323539333065633437333331353665366433386638353961663364633834 +65353166633039623837623939303763633561363931323561346165343437646134336665396264 +38306337613864343534353637376361323034653439663537316363643737316235623534396634 +30636435643464313834316133663764356133353666353036643032323263373439353363383465 +31343163653863633237633132646434663130356538313835663238383535396263616162346564 +61613132366136333436343434633139383536303061366338393637353764363164333761616635 +64316565636666333466313762373361326535383864666364663232643435323331353737316330 +31633738323533636262626264653632313638356239353235623636663638353334363934656264 +61666439623937363464316232343561623634323031326139356434363864323836313138303032 +33663965613961643434376333303332356139373066656163363737313833336634376533636130 +61353463366664356463363336633036386562646539386331633932613935643730636566313466 +31356263666631363635646466386233356133653834666133313136623165353238343731343161 +33326337663235643164663664326638656134663332376136383164336637333033306263396464 +65626431366564653832386462643431303137666532396635323966303734386365343632663733 +30313530323863393737623162646537383864353733633933613336323365363763313832623462 +66363563356462393938376632333338623232656130643666313463313765626431396132613165 +66663433303234383864323632303531616436663461396335333966303932666165353630316461 +39386438353965323036643433613232663935663938653030613662623136376365393666363535 +32333334333161656636343563663732626230353964626433633236396233613362393636343161 +63303863323961323466396532396132616437626365303164303832303834383466383437303966 +33656635623966326662656630363730326664643637383636616334393363636261663465336536 +65346438366165666331386531653835306461636431616539333366393239633536316166373862 +32613339366334343131636163376438646233393137626165383334306133653733646232306339 +66393435663132636235376130356132303037336535386461326365616137363332373262323634 +61303966376364623666366561306364356163353865613466636137323435313165663466366362 +31386462363739303430326438643436333635373236623237656230616631666131366462336561 +31623063393163623238386336336131353437373565333338643031363661366231383932626434 +66663836323666663865303938633338663563643130353334623831363961326435663765313634 +64356430383164636430653765653565653262336565353136633239393666316666633063323234 +66346537316432346531313362386533653765343561663338333734323437343935376239646464 +35336534366331393663663766616365666130353530306331336135373064373833343731643932 +32666362336236643237326636373739653032666165316164633138323233623936366462373964 +32323862633830623261623961316161323430396631633639396332393862363231313361326364 +30376331376238303934386562326266666264663434373564353839363634316664356531356632 +66376264336232346661633133373935633065363930636162326531303438366263346265393634 +61643833373630613836333337623931376231636533653536393563316436366665383030393637 +63373864376535663765313136363134656563303731666633653362366364316139313337633663 +62626139393430326136396331633532613135313762616133323665396531366634643932666266 +30663863653466643765663739666431663838376633316165353637663961376466376265363132 +61656363646534306165376134353265653230643039613964313462336161366636316433333930 +63393264396230363164373437306330346563346164633933393461363035643034346461336466 +38353336303264383166393633306663646265373932323663633032333162643831323331333262 +63313631653031633532626133303537616338306132396664643166333431663434653934383739 +39303738323966616262316164643935393261396361663066643239646638636431356164346635 +30653261656537613733313064643038396133363965656533666134343332663238626665626461 +32636162353337363762343832323234646564383565623065323136363536383766636539346339 +61643537653763383739613036363430316533363531383734633739323765613137623538636133 +31323731393461363765363231373061373831633433663036363539633230653430396335393037 +33646138366462323963653038356239656461323839636332313162333335313461636433653539 +34623161666333313038303934636539343334623730653762346334373533623264323532336433 +32616465353133336361323933393561356238643834646534376265353261316434386638373639 +39356264353833343635386536313961326662623435323236323835323266353131316462613031 +62326238373238633334646134303739386439613566336331623664313139303534303839626564 +36636237396666383966386163346631623534303733313531376432306466393330393061376132 +64303435653935303164336336653132653863366662616163326239383435306639333863616134 +61356164323966626161363864396533363130613434363365383262663937393439346665396134 +63383837643633323234363239626431363835666635366230353464373136336432366564636561 +65663236373664623039356165303365643439656561663236666337366535343731343163646636 +61373463393734313666313932613039336265343132613963373164663932623130333630336562 +32353662353439323338316335346261663831393861393637323734633939666534323532633437 +66626330626634666531346264373561353963636334333139383735666131373265313136396462 +36316266376338363965383731626130666237636638326637303330663535373461363265303239 +35383161326131353534313530623734346334383231373762303739636230613136303166356264 +64303937336461346162393365356436633239306434303163336161383236646265636331363333 +63626332643662613839323036383134346662393565656630656262663061656661663533333763 +63663934666361313961343338616532393661626365636134346331383861336431646662663033 +66316438646463636637656532376366636632326635666134366136656534366339306362383538 +32363763313265303035663436663132383539393166363961316631336464323363353466336161 +61303366306436383266616261316630396563373836663433663366616436613333323365653463 +65373730306234353331626634306162353930313539653738353434613461316665343961613338 +65613864643637363464616664663463303237313238373565393431653632366137366530643836 +61313264383238396430653631353431313562386561653362373830376261386538363666303039 +66316632633665393961346339313338363131373361653638633435383033393239383965636631 +32383865316662643636316362373065323364666262343139656363376265653638383134373837 +35383264633164326533633731306239333837353632326137383237333464356133353462613136 +30646436333935363263626131613864356163363633326662623465336362623061383166326163 +37356430663830366434653131653636626466353665666330343566306364303065393136383331 +34336239646563323664663733643130373463386135626561333361313039343932353530643061 +33323361383337623162363061343632643137623837386433366637356562346561373039666636 +65656534326632666532313333616236613032333536303664353564343962346163636132626130 +32373635323739353231336139643461363962333562316364666161643064356264353762383565 +64666663653761393135343238373064323064383263313330613265313230356133663964383336 +65633761643631656363323732646463623963616466396137383131363532393838376335303362 +31323538333264313163336362306461313538316330393261333061323162303330313536616662 +30373431343761313230616563303565326639353963326666313137633233303034333266613735 +63353835333739356231323538363336346337346236373661633132313331333865623238323436 +63373936356431316136376163383032653733313331666332383135313934336664633263613232 +30626339393365643735653761316636353632613663646539363238356332346230346638383432 +31346133346430306265353064643361396539613938326434366464303537643733613961313163 +33363564646230613865306335306262343735396330653864633039613565323033646436323334 +32393130613864646637616436653965356462636530653961653231383932626263336466613334 +37356132383361313762623637303337633763373363643938623230666230306238313135626434 +62663264373532656430393534643030386162656665323432323238626632363731613061646664 +33333033316664363365666164613333613439376462663565366132623833623432656537373737 +38646238653237633338363935626462383335333637356638396262313461336532646233353733 +39383038643039323062643165333039636233643034653930366133356135653839626136323336 +63646265366537373361386336396137383461336337323562646232653262396437366162383838 +37616330656562323238633362376637613964393630633037303039666663346634343133663831 +34383139656561393363323034343337643935353864343165626563646330656632646135333438 +34306533326364353561623762613162363431306364656634356138306233633565643938353432 +39616165316162383830626436623864316331356263626231376261356235666561323862323962 +38316237613065663233636133323566633335316361353838336466643966303530363163373431 +36613539323066643936353039353430643734303465343866303334366665623134373961346236 +32356137613036313066313864353766623961383738366262356364313061383735666436636430 +66353031323533333832643231343137653061623362306466386239396662383332343838613033 +36653263653038656562636666376663323363323239383630643663313539303461323434636139 +62626333313739336136623962343235636332623065626432326637313538346162663762656632 +61313332323338633938313164383536316161333263643037373863376634366538393065663936 +62623134373132363162386266656164653964353462303339376365316630333461363761643661 +65663064303666303236633630623230356432323931396138623862323863616535316165636463 +63373539396638373463393732653361313331666130636264616665636238623461616238613061 +32333539643961303566383365656165306264653536303365383639663738323666333030323631 +65363531346334386134633137393862316233633864363364376436663165363166366432373334 +64643338373539313131646138633963653033386236313663336266323664363661663535363539 +66373336626265636536313161386366623031633966393832633664313732643164623264373032 +37633666623534613137363765326136363462643735303130333532653433633831643736653763 +34656337656139633837343433663366303737316665613930383139646663373434316437346665 +63616130366232633835303439613638653362633764313333366435363734373536346230343031 +62623564646434396136303062653838616430306335393138316332626436353335393131303235 +33376664613931643536656238343366383332663737383331316339313330306364303861303634 +61356239356437636532623164656430356437396264333034656337366535656232353533623764 +65396639626635363032366232316561393365643737306135343134313032653963376564656265 +33356261643765336463366537643262323934383461613333633931333162313533613030386431 +32353263326537363034663138383163656461623462613738616463363830343138653138386532 +33626631623132313466633664623461643137643734326133353936653735306637343134366262 +35643864663737306230323331383637396165323864636533656331326165353264643061376661 +39313661333361616434333362643936663235363132353834373134303861656138346334613864 +32373163653362326537623335376236363962623561396430623439393038303838336166646234 +30323561333963306239623163373637666231656238623935313766393438646539326233656232 +63373136306635326263343333623461396333303339336561636132383564346163393263633464 +37313030653163643561373937663832613461383338383062636534303137656236323631613233 +62636639386236653737393631666635623832316361306165343035386536376261633636323332 +62383565663735613034323664363364653662393966323463613538393061336535393736663636 +39343939633932656633383030633164356130346138336263666235666632386235383162316161 +63393338313236353565393431623961636462336364313735623235343363373332626535393466 +35613730356164326662393362663735636430666337366633313430643631306462316133366636 +32393262326238656564663963613365613661343166383863623230643336616631656265343133 +33396435353933343963616263663337636264396136373163633865316333663839346665313036 +36353562346334623535653132313666396331383764366462623035333731336639346361333835 +36613435313839623233626633306562363461353036343639633362636537393362613432666232 +31623232653566636261623966356362333532303536353439326439643464623165626635333262 +38643236303534323238626635653332313966616566636331376438613633363432383330663037 +32376330336130653737623535333730643963356264653963313232326564303465323563326131 +38306164333061633033616561376263346635663237383133393961646330373039313265386661 +61326130653736643735646632343635343866346463383236343334653865643365633338303966 +63326536306166373334643662633434323034636538663866383736613736656437343439326237 +66653263613763383933376235636639613930616166313663393130323662636437396461393939 +37313431333237396465393466633835666365333439633266353634306431646564333262656232 +37363534373233646235303937333933346533306637333431396437633438396539313061313338 +35376137666132663566653939666532323232303466333536656364653337653337353063393939 +39343961643337353238383765396337653631396635323564333861366439393938623964363234 +34383262633838646363663738663461396238636661643466323464323732316135663165363236 +64633435393364323734356635323237306536383362613665353531383264386538353263396362 +31353862376166346538663231663732353066393736666266386231353130306232393864316137 +64643666303061653266343437313034633662323335383761386237336435633738646665633931 +36326532613466383365346138323466333432343166343037316663646562336661343739623738 +30326362366437323436376634316664633931646661323937333936666165303630666432353662 +32643931356664643266353530396436343730353766386663323735656132386363373063313936 +37383330623563363763343331623961333238633838616234616134373538636338336437353135 +35356538633239643233353162366530343739653030363365303134343631326265653664313462 +38333530303037303262323466626261313961386233656137376661656663303632666232336537 +36616266646534333238616434383564343938373132663839636165323061376163353536393361 +65653430366333623364343631333961333736633362623934653264323266666666653763363039 +34303966376662626635353839346163633936656366646236313239643239363066636237623832 +64633135356234623161303439373238303361326438656338626432323732316239656664336533 +31353666363131666537386563613062633861376230376339333836383538313666616338333263 +64663365333337626162353831363032326363373064303961303037383565663265386562383663 +34666637666162366337336637666238393561613665643161643564316539333038373839346132 +63303066323832653364623439313631386663653865646565383166336531306332303330666561 +63643933353332623564383933653964643463653930626434326562643230616432666638653665 +31386664643737336262643438353532346632343466646136613232393838363863643833663764 +38346530626339646462343662653863613765633863643564323866363333636431643734356161 +31663837346131646532633838393766336261613061326264333937393334356338383633383961 +37356539663132653166316362303064373431633036366465306263376239313336666361636531 +30633865383535643537333735323662393936613935373539306435653033616133366663623533 +61346630383433633634353431396530343032366535653466383162383134366234653961366634 +31373739333261656165376531616536623032636530303361333266643561333064306334393065 +65303639643137343933653538656336363932333465373432636161386232336339616363343539 +66616132343535323936616438313335313262363963393731336262333764303738316537386661 +36383161616462323431666464633464656561623638326163313132353534643565356635353130 +32653835306237663966376561623233636433613235343761336233643631313235346137316361 +65326338613066653930643065643465343762333661633266393232306563303739333539306463 +64646139623631613863376465616163643862333030373735376331336137613131346133393834 +32636563646664636533646262636532373562366463393633336564373833303337613666336432 +31613038646531663163336437373262396138363764383863633234643762616363643761613835 +30666630313937643833353935306165656435333039306531386365386437656238376339343634 +66626161643335393263336530646166623135623132353432323964376266643161313033626338 +39323832393562613164333538656236663139623633636461323833323236376238393062306263 +36343862623634323332643765663833396437353936393262376531366565393239643837393737 +61636461616166323431393933373537366665363630336635656533623731363132663337663733 +61393936386362373661323931396539393438373330396633326138303237366138613039393032 +33633234623034396231643266653564316138666139343062623361653530633065393736303064 +32643633623939326563363633346331633732616364333236653734626635646665336665393539 +61326363633666326631616436633236363863393364326130313632613064353230353065623965 +37366539346230353665653933653862386330313530353331396637353464633161663064366266 +61303265663237333065376665663138313737363432663839623032636338346533393763666531 +33343162373132623464653763343732653361663366363965353836363939306230656437333965 +31666463383664613966646132666431383665373061326361646463383262373061316636343734 +33633664383633643031383561646139656363653835666234616438626165613731396535333732 +38366136623437356532373436616237396135313732376635643138623537356138653738353537 +33656334616666353836633132303039346532323963363431383339666161623931623563623937 +39303134663565393930613939376564363332373561376534313537646265303239383138663336 +31316133326661303366646261326433363539663365363763643537353165336164333066636431 +33363738376465303435373064643336353962343732376461663638303566356435313935383830 +37653638336338306538383232626231666562333532356532383634313437626466376564303863 +39656333383835393035323966623834326232373136623963323239313134326335323338343661 +34313233323466383262656238623839616636396463616463366666656432643331333761613134 +62366531653639383265393833633132373361613833653239373665373537373231363361333062 +38356238646531383736363530616439363930363733383532303330386330353161643639363934 +66373436613633626139613033633432623766363633646632323632653264393435363336333731 +63366330346530326535663734323066613736383063616337616265396664633938633837636461 +33303633666361313930613866356233373335633066373939643437356537313861323534333735 +37656630653336313139616536363666303737643432316566633962633933656137633061336139 +36623636383237666139623931316535353631306665626131346361303164386663653534613530 +38366130343034313734643735373363643234373839623962653562623932323832386364616138 +63306265616132346330336139306262623435393739643131363639613235366662616266343338 +65323436383137313065303534376236663537376436383133343533663839316561313738376635 +61303965356334326138666165366338663632333366356465313765373165356530303038346565 +37383835636263306438373763393230323231326366333333353762333062373039316661323330 +34313564363766636439636465306133366161313033323331663765626338396138323930306365 +32636432653330383564633464396536333633363831336635386537316334323163656663653532 +36373430373363633333343134323136663361643934363265313464366630346338303535363233 +64353763316133666133656335633466633332353461326538653733666436383261616365363233 +36373139363533343130383734666161616439393539633032636338303636633837393338393366 +62373732303433663039353161356539646539613365616334643530663662363266316163353466 +31633563363966366636616135333831326262666135633464343334343564626538666236393131 +35353564616630653635373964656461376266323361313861663165373766353834313836303866 +61643532303562356135633837323136373432313131393065393531346131663264383966326562 +37643464326430313734653133313264336634363431633136336334333939366666633761386531 +38336664313031636661313837646461653133616139616265336530393531653237316165363062 +35643337393037653665336330666637656532366135613635393431306631393566376561306234 +37623265303633316434613938663366623966663434393161653635303033633862633533366364 +37343737393338323936333035333366613166653165363737343136383036386264613864376365 +30376261303665323439623866326633646337626534663031386164323362333230346134623466 +65323735646465346663643266643564633434316131353561303966653062616332306436343234 +63313131643436646237343231363331356632616234346362383431666566383263383463323734 +64666332363130353332643731623830616430636233366237613139623864646338303561653230 +39663039323939643939303130633333313565366463636232386235363435333534356135666463 +33666436383364376164396266303339393234396538313566316336393264656135613132643366 +63353037306137313436336631303436363665333633623366326538613834663337313462343530 +62333936396232666633303062623934653131616364363139656539386666663063623763643465 +35333035363738663731613637633337346361396363343334393637373434346564393766393633 +62633433346563396163356266366136353965306662316232666261393336626537393130643163 +35656333313864623334306437663537623831656536633232333433653336616464383561303937 +35376466333964396430346161306633393862366262623134333963656234633061356264333235 +34626438326536396663323338613762613536376234376535626134386237373963383865376238 +62396636653437396564303163613366353766333936303438393062646362373535306136316162 +34396136396636326632663033343538323530663764636166306130306164613936626139323939 +39643831613537616263323661336232336661643263383534616535366163323030626261346232 +30633265323734636330383037656638656134323438666339656164306634313364636232353263 +37623364376435363431 diff --git a/changehc/delphi_changehc/run.py b/changehc/delphi_changehc/run.py index 92a03e6c5..9c15b221d 100644 --- a/changehc/delphi_changehc/run.py +++ b/changehc/delphi_changehc/run.py @@ -25,7 +25,7 @@ def retrieve_files(params, filedate, logger): if files["denom"] is None: ## download recent files from FTP server - logger.info("downloading recent files through SFTP") + logger.info("Downloading recent files through SFTP") download_counts(filedate, params["indicator"]["input_cache_dir"], params["indicator"]["ftp_conn"]) denom_file = "%s/%s_Counts_Products_Denom.dat.gz" % (params["indicator"]["input_cache_dir"],filedate) @@ -157,18 +157,20 @@ def run_module(params: Dict[str, Dict[str, Any]]): startdate, enddate = process_dates(params, startdate_dt, enddate_dt) - logger.info("generating signal and exporting to CSV", - first_sensor_date = startdate, - last_sensor_date = enddate, - drop_date = dropdate, - n_backfill_days = n_backfill_days, - n_waiting_days = n_waiting_days, - geos = params["indicator"]["geos"], - export_dir = params["common"]["export_dir"], - parallel = params["indicator"]["parallel"], - weekday = params["indicator"]["weekday"], - types = params["indicator"]["types"], - se = params["indicator"]["se"]) + logger.info( + "Generating signal and exporting to CSV", + first_sensor_date=startdate, + last_sensor_date=enddate, + drop_date=dropdate, + n_backfill_days=n_backfill_days, + n_waiting_days=n_waiting_days, + geos=params["indicator"]["geos"], + export_dir=params["common"]["export_dir"], + parallel=params["indicator"]["parallel"], + weekday=params["indicator"]["weekday"], + types=params["indicator"]["types"], + se=params["indicator"]["se"], + ) ## start generating stats = [] @@ -176,9 +178,9 @@ def run_module(params: Dict[str, Dict[str, Any]]): for numtype in params["indicator"]["types"]: for weekday in params["indicator"]["weekday"]: if weekday: - logger.info("starting weekday adj", geo = geo, numtype = numtype) + logger.info("Starting weekday adj", geo_type=geo, numtype=numtype) else: - logger.info("starting no adj", geo = geo, numtype = numtype) + logger.info("Starting no adj", geo_type=geo, numtype=numtype) su_inst = CHCSensorUpdater( startdate, enddate, @@ -211,7 +213,7 @@ def run_module(params: Dict[str, Dict[str, Any]]): ) stats.extend(more_stats) - logger.info("finished processing", geo = geo) + logger.info("Finished processing", geo_type=geo) elapsed_time_in_seconds = round(time.time() - start_time, 2) min_max_date = stats and min(s[0] for s in stats) diff --git a/changehc/delphi_changehc/sensor.py b/changehc/delphi_changehc/sensor.py index 0449f07df..9a1fd29e0 100644 --- a/changehc/delphi_changehc/sensor.py +++ b/changehc/delphi_changehc/sensor.py @@ -118,10 +118,10 @@ def fit(y_data, first_sensor_date, geo_id, logger, num_col="num", den_col="den") se_valid = valid_rates.eval('sqrt(rate * (1 - rate) / den)') rate_data['se'] = se_valid - logger.debug("{0}: {1:.3f},[{2:.3f}]".format( - geo_id, rate_data['rate'][-1], rate_data['se'][-1] - )) - return {"geo_id": geo_id, - "rate": 100 * rate_data['rate'], - "se": 100 * rate_data['se'], - "incl": include} + logger.debug( + ".fit() DEBUG - last rate/se for geo", + geo_value=geo_id, + value=rate_data["rate"][-1], + se=rate_data["se"][-1], + ) + return {"geo_id": geo_id, "rate": 100 * rate_data["rate"], "se": 100 * rate_data["se"], "incl": include} diff --git a/changehc/delphi_changehc/update_sensor.py b/changehc/delphi_changehc/update_sensor.py index edae85517..7c78dc020 100644 --- a/changehc/delphi_changehc/update_sensor.py +++ b/changehc/delphi_changehc/update_sensor.py @@ -41,7 +41,7 @@ def write_to_csv(df, geo_level, write_se, day_shift, out_name, logger, output_pa assert df[suspicious_se_mask].empty, " se contains suspiciously large values" assert not df["se"].isna().any(), " se contains nan values" if write_se: - logger.info("========= WARNING: WRITING SEs TO {0} =========".format(out_name)) + logger.info("WARNING: WRITING SEs", filename=out_name) else: df["se"] = np.nan @@ -49,9 +49,7 @@ def write_to_csv(df, geo_level, write_se, day_shift, out_name, logger, output_pa suspicious_val_mask = df["val"].gt(90) if not df[suspicious_val_mask].empty: for geo in df.loc[suspicious_val_mask, "geo_id"]: - logger.warning("value suspiciously high, {0}: {1}".format( - geo, out_name - )) + logger.warning("Value suspiciously high", geo_value=geo, filename=out_name) dates = create_export_csv( df, @@ -62,10 +60,8 @@ def write_to_csv(df, geo_level, write_se, day_shift, out_name, logger, output_pa sensor=out_name, write_empty_days=True ) - logger.debug("wrote {0} rows for {1} {2}".format( - df.size, df["geo_id"].unique().size, geo_level - )) - logger.debug("wrote files to {0}".format(output_path)) + logger.debug("Wrote rows", num_rows=df.size, geo_type=geo_level, num_geo_ids=df["geo_id"].unique().size) + logger.debug("Wrote files", export_dir=output_path) return dates @@ -148,8 +144,9 @@ def geo_reindex(self, data): geo = self.geo gmpr = GeoMapper() if geo not in {"county", "state", "msa", "hrr", "nation", "hhs"}: - self.logger.error("{0} is invalid, pick one of 'county', " - "'state', 'msa', 'hrr', 'hss','nation'".format(geo)) + self.logger.error( + "Geo is invalid, pick one of 'county', " "'state', 'msa', 'hrr', 'hss','nation'", geo_type=geo + ) return False if geo == "county": data_frame = gmpr.fips_to_megacounty(data, @@ -224,7 +221,7 @@ def update_sensor(self, dfs.append(res) else: n_cpu = min(10, cpu_count()) - self.logger.debug("starting pool with {0} workers".format(n_cpu)) + self.logger.debug("Starting pool", n_workers=n_cpu) with Pool(n_cpu) as pool: pool_results = [] for geo_id, sub_data in data_frame.groupby(level=0,as_index=False): diff --git a/changehc/setup.py b/changehc/setup.py index d95beb771..f386e9613 100644 --- a/changehc/setup.py +++ b/changehc/setup.py @@ -6,6 +6,7 @@ "covidcast", "darker[isort]~=2.1.1", "delphi-utils", + "mock", "moto~=4.2.14", "numpy", "pandas", diff --git a/changehc/tests/test_update_sensor.py b/changehc/tests/test_update_sensor.py index 7ef25a608..d2e7ee2f3 100644 --- a/changehc/tests/test_update_sensor.py +++ b/changehc/tests/test_update_sensor.py @@ -9,6 +9,7 @@ import pandas as pd import numpy as np from boto3 import Session +from delphi_utils import get_structured_logger from moto import mock_s3 import pytest @@ -28,7 +29,7 @@ DENOM_FILEPATH = PARAMS["indicator"]["input_denom_file"] DROP_DATE = pd.to_datetime(PARAMS["indicator"]["drop_date"]) OUTPATH="test_data/" -TEST_LOGGER = logging.getLogger() +TEST_LOGGER = get_structured_logger() class TestCHCSensorUpdater: """Tests for updating the sensors.""" diff --git a/changehc/version.cfg b/changehc/version.cfg index f5c28d2cd..ed4b085f1 100644 --- a/changehc/version.cfg +++ b/changehc/version.cfg @@ -1 +1 @@ -current_version = 0.3.55 +current_version = 0.3.56 diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py index 5c9019035..20fd0d953 100644 --- a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -58,7 +58,7 @@ def download(ftp_credentials, out_path, logger, issue_date=None): current_time = issue_date if issue_date else datetime.datetime.now() seconds_in_day = 24 * 60 * 60 - logger.info("starting download", time=current_time) + logger.info("Starting download") # open client client = paramiko.SSHClient() diff --git a/claims_hosp/delphi_claims_hosp/indicator.py b/claims_hosp/delphi_claims_hosp/indicator.py index 4ad3ef7df..c5ac4e886 100644 --- a/claims_hosp/delphi_claims_hosp/indicator.py +++ b/claims_hosp/delphi_claims_hosp/indicator.py @@ -143,7 +143,10 @@ def fit(y_data, first_date, geo_id, num_col="num", den_col="den"): se_valid = valid_rates.eval('sqrt(rate * (1 - rate) / den)') rate_data['se'] = se_valid - logging.debug("%s: %05.3f, [%05.3f]", - geo_id, rate_data['rate'][-1], rate_data['se'][-1]) - return {"geo_id": geo_id, "rate": 100 * rate_data['rate'], - "se": 100 * rate_data['se'], "incl": include} + logging.debug( + ".fit() DEBUG - last rate/se for geo", + geo_value=geo_id, + value=rate_data["rate"][-1], + se=rate_data["se"][-1], + ) + return {"geo_id": geo_id, "rate": 100 * rate_data["rate"], "se": 100 * rate_data["se"], "incl": include} diff --git a/claims_hosp/delphi_claims_hosp/modify_claims_drops.py b/claims_hosp/delphi_claims_hosp/modify_claims_drops.py index 0ab93ebcc..19a962884 100644 --- a/claims_hosp/delphi_claims_hosp/modify_claims_drops.py +++ b/claims_hosp/delphi_claims_hosp/modify_claims_drops.py @@ -57,5 +57,5 @@ def modify_and_write(data_path, logger, test_mode=False): dfs_list.append(dfs) else: dfs.to_csv(out_path, index=False) - logger.info(f"Wrote {out_path}") + logger.info("Wrote modified csv", filename=out_path) return files, dfs_list diff --git a/claims_hosp/delphi_claims_hosp/run.py b/claims_hosp/delphi_claims_hosp/run.py index 0d24192c0..3a8538bb2 100644 --- a/claims_hosp/delphi_claims_hosp/run.py +++ b/claims_hosp/delphi_claims_hosp/run.py @@ -130,9 +130,9 @@ def run_module(params, logger=None): for geo in params["indicator"]["geos"]: for weekday in params["indicator"]["weekday"]: if weekday: - logger.info("starting weekday adj", geo = geo) + logger.info("Starting weekday adj", geo_type=geo) else: - logger.info("starting no weekday adj", geo = geo) + logger.info("Starting no weekday adj", geo_type=geo) signal_name = Config.signal_weekday_name if weekday else Config.signal_name if params["indicator"]["write_se"]: @@ -140,7 +140,7 @@ def run_module(params, logger=None): "supply obfuscated prefix in params.json" signal_name = params["indicator"]["obfuscated_prefix"] + "_" + signal_name - logger.info("Updating signal name", signal_name = signal_name) + logger.info("Updating signal name", signal=signal_name) updater = ClaimsHospIndicatorUpdater( startdate, enddate, @@ -149,16 +149,16 @@ def run_module(params, logger=None): params["indicator"]["parallel"], weekday, params["indicator"]["write_se"], - signal_name + signal_name, + logger, ) updater.update_indicator( claims_file, params["common"]["export_dir"], - logger, ) max_dates.append(updater.output_dates[-1]) n_csv_export.append(len(updater.output_dates)) - logger.info("finished updating", geo = geo) + logger.info("Finished updating", geo_type=geo) # Remove all the raw files for fn in os.listdir(params["indicator"]["input_dir"]): diff --git a/claims_hosp/delphi_claims_hosp/update_indicator.py b/claims_hosp/delphi_claims_hosp/update_indicator.py index df3f3308f..5ba8ddd22 100644 --- a/claims_hosp/delphi_claims_hosp/update_indicator.py +++ b/claims_hosp/delphi_claims_hosp/update_indicator.py @@ -7,7 +7,6 @@ """ # standard packages -import logging from multiprocessing import Pool, cpu_count # third party @@ -28,8 +27,7 @@ class ClaimsHospIndicatorUpdater: # pylint: disable=too-many-instance-attributes, too-many-arguments # all variables are used - def __init__(self, startdate, enddate, dropdate, geo, parallel, weekday, - write_se, signal_name): + def __init__(self, startdate, enddate, dropdate, geo, parallel, weekday, write_se, signal_name, logger): """ Initialize updater for the claims-based hospitalization indicator. @@ -53,6 +51,7 @@ def __init__(self, startdate, enddate, dropdate, geo, parallel, weekday, # init in shift_dates, declared here for pylint self.burnindate, self.fit_dates, self.burn_in_dates, self.output_dates = \ [None] * 4 + self.logger = logger assert ( self.startdate > (Config.FIRST_DATA_DATE + Config.BURN_IN_PERIOD) @@ -114,9 +113,9 @@ def geo_reindex(self, data): elif self.geo == "hrr": data_frame = data # data is already adjusted in aggregation step above else: - logging.error( - "%s is invalid, pick one of 'county', 'state', 'msa', 'hrr', 'hhs', nation'", - self.geo) + self.logger.error( + "Geo is invalid, pick one of 'county', 'state', 'msa', 'hrr', 'hhs', nation'", geo_type=self.geo + ) return False unique_geo_ids = pd.unique(data_frame[self.geo]) @@ -133,7 +132,7 @@ def geo_reindex(self, data): data_frame.fillna(0, inplace=True) return data_frame - def update_indicator(self, input_filepath, outpath, logger): + def update_indicator(self, input_filepath, outpath): """ Generate and output indicator values. @@ -159,7 +158,7 @@ def update_indicator(self, input_filepath, outpath, logger): ["num"], Config.DATE_COL, [1, 1e5], - logger, + self.logger, ) if self.weekday else None @@ -182,7 +181,7 @@ def update_indicator(self, input_filepath, outpath, logger): valid_inds[geo_id] = np.array(res.loc[final_output_inds, "incl"]) else: n_cpu = min(Config.MAX_CPU_POOL, cpu_count()) - logging.debug("starting pool with %d workers", n_cpu) + self.logger.debug("Starting pool", n_workers=n_cpu) with Pool(n_cpu) as pool: pool_results = [] for geo_id, sub_data in data_frame.groupby(level=0, as_index=False): @@ -217,7 +216,7 @@ def update_indicator(self, input_filepath, outpath, logger): } self.write_to_csv(output_dict, outpath) - logging.debug("wrote files to %s", outpath) + self.logger.debug("Wrote files", export_dir=outpath) def write_to_csv(self, output_dict, output_path="./receiving"): """ @@ -229,8 +228,7 @@ def write_to_csv(self, output_dict, output_path="./receiving"): """ if self.write_se: - logging.info("========= WARNING: WRITING SEs TO %s =========", - self.signal_name) + self.logger.info("WARNING: WRITING SEs", signal=self.signal_name) geo_level = output_dict["geo_level"] dates = output_dict["dates"] @@ -255,7 +253,7 @@ def write_to_csv(self, output_dict, output_path="./receiving"): assert not np.isnan(val), "value for included value is nan" assert not np.isnan(se), "se for included rate is nan" if val > 90: - logging.warning("value suspicious, %s: %d", geo_id, val) + self.logger.warning("Value suspicious", geo_type=geo_level, geo_value=geo_id, value=val) assert se < 5, f"se suspicious, {geo_id}: {se}" if self.write_se: assert val > 0 and se > 0, "p=0, std_err=0 invalid" @@ -267,4 +265,4 @@ def write_to_csv(self, output_dict, output_path="./receiving"): "%s,%f,%s,%s,%s\n" % (geo_id, val, "NA", "NA", "NA")) out_n += 1 - logging.debug("wrote %d rows for %d %s", out_n, len(geo_ids), geo_level) + self.logger.debug("Wrote rows", num_rows=out_n, geo_type=geo_level, num_geo_ids=len(geo_ids)) diff --git a/claims_hosp/tests/test_update_indicator.py b/claims_hosp/tests/test_update_indicator.py index 1471be655..5ca527287 100644 --- a/claims_hosp/tests/test_update_indicator.py +++ b/claims_hosp/tests/test_update_indicator.py @@ -51,7 +51,8 @@ def test_shift_dates(self): self.parallel, self.weekday, self.write_se, - Config.signal_name + Config.signal_name, + TEST_LOGGER ) ## Test init assert updater.startdate.month == 2 @@ -72,7 +73,8 @@ def test_geo_reindex(self): self.parallel, self.weekday, self.write_se, - Config.signal_name + Config.signal_name, + TEST_LOGGER ) updater.shift_dates() data_frame = updater.geo_reindex(self.small_test_data.reset_index()) @@ -90,13 +92,13 @@ def test_update_indicator(self): self.parallel, self.weekday, self.write_se, - Config.signal_name + Config.signal_name, + TEST_LOGGER ) updater.update_indicator( DATA_FILEPATH, - td.name, - TEST_LOGGER + td.name ) assert len(os.listdir(td.name)) == len( @@ -112,7 +114,8 @@ def test_write_to_csv_results(self): self.parallel, self.weekday, self.write_se, - Config.signal_name + Config.signal_name, + TEST_LOGGER ) res0 = { @@ -192,7 +195,8 @@ def test_write_to_csv_with_se_results(self): self.parallel, True, True, - signal_name + signal_name, + TEST_LOGGER ) res0 = { @@ -243,7 +247,8 @@ def test_write_to_csv_wrong_results(self): self.parallel, self.weekday, self.write_se, - Config.signal_name + Config.signal_name, + TEST_LOGGER ) res0 = { diff --git a/claims_hosp/version.cfg b/claims_hosp/version.cfg index f5c28d2cd..ed4b085f1 100644 --- a/claims_hosp/version.cfg +++ b/claims_hosp/version.cfg @@ -1 +1 @@ -current_version = 0.3.55 +current_version = 0.3.56 diff --git a/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py b/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py index 9d51768be..6cb42364a 100644 --- a/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py +++ b/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py @@ -59,7 +59,7 @@ def download(ftp_credentials, out_path, logger, issue_date=None): else: current_time = datetime.datetime.strptime(issue_date, "%Y-%m-%d").replace(hour=23, minute=59, second=59) - logger.info("starting download", time=current_time) + logger.info("Starting download") seconds_in_day = 24 * 60 * 60 # open client diff --git a/doctor_visits/delphi_doctor_visits/modify_claims_drops.py b/doctor_visits/delphi_doctor_visits/modify_claims_drops.py index daed93d58..3be9393e4 100644 --- a/doctor_visits/delphi_doctor_visits/modify_claims_drops.py +++ b/doctor_visits/delphi_doctor_visits/modify_claims_drops.py @@ -48,5 +48,5 @@ def modify_and_write(f, logger, test_mode=False): if not test_mode: dfs.to_csv(out_path, index=False) - logger.info(f"Wrote {out_path}") + logger.info("Wrote modified csv", filename=out_path) return dfs diff --git a/doctor_visits/delphi_doctor_visits/patch.py b/doctor_visits/delphi_doctor_visits/patch.py index 32b6d308f..32c62dd6b 100644 --- a/doctor_visits/delphi_doctor_visits/patch.py +++ b/doctor_visits/delphi_doctor_visits/patch.py @@ -45,16 +45,19 @@ def patch(): start_issue = datetime.strptime(params["patch"]["start_issue"], "%Y-%m-%d") end_issue = datetime.strptime(params["patch"]["end_issue"], "%Y-%m-%d") - logger.info(f"""Start patching {params["patch"]["patch_dir"]}""") - logger.info(f"""Start issue: {start_issue.strftime("%Y-%m-%d")}""") - logger.info(f"""End issue: {end_issue.strftime("%Y-%m-%d")}""") + logger.info( + "Starting patching", + patch_directory=params["patch"]["patch_dir"], + start_issue=start_issue.strftime("%Y-%m-%d"), + end_issue=end_issue.strftime("%Y-%m-%d"), + ) makedirs(params["patch"]["patch_dir"], exist_ok=True) current_issue = start_issue while current_issue <= end_issue: - logger.info(f"""Running issue {current_issue.strftime("%Y-%m-%d")}""") + logger.info("Running issue", issue_date=current_issue.strftime("%Y-%m-%d")) params["patch"]["current_issue"] = current_issue.strftime("%Y-%m-%d") diff --git a/doctor_visits/delphi_doctor_visits/run.py b/doctor_visits/delphi_doctor_visits/run.py index 3c941534a..2dccffc8c 100644 --- a/doctor_visits/delphi_doctor_visits/run.py +++ b/doctor_visits/delphi_doctor_visits/run.py @@ -88,32 +88,33 @@ def run_module(params, logger=None): # pylint: disable=too-many-statements startdate_dt = enddate_dt - timedelta(days=n_backfill_days) enddate = str(enddate_dt.date()) startdate = str(startdate_dt.date()) - logger.info("drop date:\t\t%s", dropdate) - logger.info("first sensor date:\t%s", startdate) - logger.info("last sensor date:\t%s", enddate) - logger.info("n_backfill_days:\t%s", n_backfill_days) - logger.info("n_waiting_days:\t%s", n_waiting_days) + + logger.info( + "Using params", + startdate=startdate, + enddate=enddate, + dropdate=dropdate, + n_backfill_days=n_backfill_days, + n_waiting_days=n_waiting_days, + export_dir=export_dir, + parallel=params["indicator"]["parallel"], + weekday=params["indicator"]["weekday"], + write_se=se, + prefix=prefix, + ) ## geographies geos = ["state", "msa", "hrr", "county", "hhs", "nation"] - - ## print out other vars - logger.info("outpath:\t\t%s", export_dir) - logger.info("parallel:\t\t%s", params["indicator"]["parallel"]) - logger.info("weekday:\t\t%s", params["indicator"]["weekday"]) - logger.info("write se:\t\t%s", se) - logger.info("obfuscated prefix:\t%s", prefix) - max_dates = [] n_csv_export = [] ## start generating for geo in geos: for weekday in params["indicator"]["weekday"]: if weekday: - logger.info("starting %s, weekday adj", geo) + logger.info("Starting with weekday adj", geo_type=geo) else: - logger.info("starting %s, no adj", geo) + logger.info("Starting with no adj", geo_type=geo) sensor = update_sensor( filepath=claims_file, startdate=startdate, @@ -137,8 +138,8 @@ def run_module(params, logger=None): # pylint: disable=too-many-statements write_to_csv(sensor, geo, se, out_name, logger, export_dir) max_dates.append(sensor.date.max()) n_csv_export.append(sensor.date.unique().shape[0]) - logger.debug(f"wrote files to {export_dir}") - logger.info("finished updating", geo = geo) + logger.debug("Wrote files", export_dir=export_dir) + logger.info("Finished updating", geo_type=geo) # Remove all the raw files for fn in os.listdir(params["indicator"]["input_dir"]): diff --git a/doctor_visits/delphi_doctor_visits/sensor.py b/doctor_visits/delphi_doctor_visits/sensor.py index b5a645ea8..91faa20cf 100644 --- a/doctor_visits/delphi_doctor_visits/sensor.py +++ b/doctor_visits/delphi_doctor_visits/sensor.py @@ -239,7 +239,7 @@ def fit(y_data, se[include] = np.sqrt( np.divide((new_rates[include] * (1 - new_rates[include])), den[include])) - logger.debug(f"{geo_id}: {new_rates[-1]:.3f},[{se[-1]:.3f}]") + logger.debug(".fit() DEBUG - last rate/se for geo", geo_value=geo_id, value=new_rates[-1], se=se[-1]) included_indices = [x for x in final_sensor_idxs if include[x]] diff --git a/doctor_visits/delphi_doctor_visits/update_sensor.py b/doctor_visits/delphi_doctor_visits/update_sensor.py index 125c0df18..4cac1e81c 100644 --- a/doctor_visits/delphi_doctor_visits/update_sensor.py +++ b/doctor_visits/delphi_doctor_visits/update_sensor.py @@ -34,7 +34,7 @@ def write_to_csv(output_df: pd.DataFrame, geo_level, se, out_name, logger, outpu output_path: outfile path to write the csv (default is current directory) """ if se: - logger.info(f"========= WARNING: WRITING SEs TO {out_name} =========") + logger.info("WARNING: WRITING SEs", filename=out_name) out_n = 0 for d in set(output_df["date"]): @@ -64,7 +64,7 @@ def write_to_csv(output_df: pd.DataFrame, geo_level, se, out_name, logger, outpu outfile.write( "%s,%f,%s,%s,%s\n" % (geo_id, sensor, "NA", "NA", "NA")) out_n += 1 - logger.debug(f"wrote {out_n} rows for {geo_level}") + logger.debug("Wrote rows", num_rows=out_n, geo_type=geo_level) def update_sensor( @@ -177,7 +177,7 @@ def update_sensor( else: n_cpu = min(10, cpu_count()) - logger.debug(f"starting pool with {n_cpu} workers") + logger.debug("Starting pool", n_workers=n_cpu) with Pool(n_cpu) as pool: pool_results = [] diff --git a/doctor_visits/version.cfg b/doctor_visits/version.cfg index f5c28d2cd..ed4b085f1 100644 --- a/doctor_visits/version.cfg +++ b/doctor_visits/version.cfg @@ -1 +1 @@ -current_version = 0.3.55 +current_version = 0.3.56 diff --git a/google_symptoms/delphi_google_symptoms/date_utils.py b/google_symptoms/delphi_google_symptoms/date_utils.py index ebfe5109a..2ad6244e9 100644 --- a/google_symptoms/delphi_google_symptoms/date_utils.py +++ b/google_symptoms/delphi_google_symptoms/date_utils.py @@ -98,7 +98,7 @@ def generate_num_export_days(params: Dict, logger) -> [int]: expected_date_diff += global_max_expected_lag if latest_date_diff > expected_date_diff: - logger.info(f"Missing dates from: {to_datetime(min(gs_metadata.max_time)).date()}") + logger.info("Missing date", date=to_datetime(min(gs_metadata.max_time)).date()) num_export_days = expected_date_diff diff --git a/google_symptoms/delphi_google_symptoms/patch.py b/google_symptoms/delphi_google_symptoms/patch.py index 01d099c4f..85df89394 100755 --- a/google_symptoms/delphi_google_symptoms/patch.py +++ b/google_symptoms/delphi_google_symptoms/patch.py @@ -58,16 +58,19 @@ def patch(params): issue_date = datetime.strptime(params["patch"]["start_issue"], "%Y-%m-%d") end_issue = datetime.strptime(params["patch"]["end_issue"], "%Y-%m-%d") - logger.info(f"""Start patching {params["patch"]["patch_dir"]}""") - logger.info(f"""Start issue: {issue_date.strftime("%Y-%m-%d")}""") - logger.info(f"""End issue: {end_issue.strftime("%Y-%m-%d")}""") + logger.info( + "Starting patching", + patch_directory=params["patch"]["patch_dir"], + start_issue=issue_date.strftime("%Y-%m-%d"), + end_issue=end_issue.strftime("%Y-%m-%d"), + ) makedirs(params["patch"]["patch_dir"], exist_ok=True) patch_dates = generate_patch_dates(params) while issue_date <= end_issue: - logger.info(f"""Running issue {issue_date.strftime("%Y-%m-%d")}""") + logger.info("Running issue", issue_date=issue_date.strftime("%Y-%m-%d")) # Output dir setup current_issue_yyyymmdd = issue_date.strftime("%Y%m%d") diff --git a/google_symptoms/delphi_google_symptoms/run.py b/google_symptoms/delphi_google_symptoms/run.py index 8303a9a8a..8ad1d6d10 100644 --- a/google_symptoms/delphi_google_symptoms/run.py +++ b/google_symptoms/delphi_google_symptoms/run.py @@ -80,10 +80,8 @@ def run_module(params, logger=None): if len(df_pull) == 0: continue for metric, smoother in product(COMBINED_METRIC, SMOOTHERS): - logger.info("generating signal and exporting to CSV", - geo_res=geo_res, - metric=metric, - smoother=smoother) + sensor_name = "_".join([smoother, "search"]) + logger.info("Generating signal and exporting to CSV", geo_type=geo_res, signal=f"{metric}_{sensor_name}") df = df_pull df["val"] = df[metric].astype(float) df["val"] = df[["geo_id", "val"]].groupby( @@ -94,9 +92,8 @@ def run_module(params, logger=None): # Drop early entries where data insufficient for smoothing df = df.loc[~df["val"].isnull(), :] df = df.reset_index() - sensor_name = "_".join([smoother, "search"]) if len(df) == 0: - logger.info("No data for %s_%s_%s", geo_res, metric.lower(), sensor_name) + logger.info("No data for signal", geo_type=geo_res, signal=f"{metric}_{sensor_name}") continue exported_csv_dates = create_export_csv( df, diff --git a/google_symptoms/version.cfg b/google_symptoms/version.cfg index f5c28d2cd..ed4b085f1 100644 --- a/google_symptoms/version.cfg +++ b/google_symptoms/version.cfg @@ -1 +1 @@ -current_version = 0.3.55 +current_version = 0.3.56 diff --git a/hhs_hosp/version.cfg b/hhs_hosp/version.cfg index f5c28d2cd..ed4b085f1 100644 --- a/hhs_hosp/version.cfg +++ b/hhs_hosp/version.cfg @@ -1 +1 @@ -current_version = 0.3.55 +current_version = 0.3.56 diff --git a/jenkins/build-indicator.sh b/jenkins/build-indicator.sh index 3d2315741..88f79424a 100755 --- a/jenkins/build-indicator.sh +++ b/jenkins/build-indicator.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Jenkins build +# Jenkins build # set -eo pipefail @@ -17,4 +17,6 @@ source env/bin/activate pip install pip==23.0.1 --retries 10 --timeout 20 pip install numpy --retries 10 --timeout 20 pip install ../_delphi_utils_python/. --retries 10 --timeout 20 -[ ! -f setup.py ] || pip install . --retries 10 --timeout 20 +if [ -f setup.py ] || [ -f pyproject.toml ]; then + pip install . --retries 10 --timeout 20 +fi diff --git a/nchs_mortality/delphi_nchs_mortality/pull.py b/nchs_mortality/delphi_nchs_mortality/pull.py index 18bbfd59a..ad54e457a 100644 --- a/nchs_mortality/delphi_nchs_mortality/pull.py +++ b/nchs_mortality/delphi_nchs_mortality/pull.py @@ -1,15 +1,17 @@ # -*- coding: utf-8 -*- """Functions for pulling NCHS mortality data API.""" +import logging from typing import Optional import numpy as np import pandas as pd +from delphi_utils import create_backup_csv +from delphi_utils.geomap import GeoMapper from sodapy import Socrata -from delphi_utils.geomap import GeoMapper +from .constants import METRICS, NEWLINE, RENAME -from .constants import METRICS, RENAME, NEWLINE def standardize_columns(df): """Rename columns to comply with a standard set. @@ -22,7 +24,13 @@ def standardize_columns(df): return df.rename(columns=dict(rename_pairs)) -def pull_nchs_mortality_data(socrata_token: str, test_file: Optional[str] = None): +def pull_nchs_mortality_data( + socrata_token: str, + backup_dir: str, + custom_run: bool, + logger: Optional[logging.Logger] = None, + test_file: Optional[str] = None, +): """Pull the latest NCHS Mortality data, and conforms it into a dataset. The output dataset has: @@ -40,6 +48,10 @@ def pull_nchs_mortality_data(socrata_token: str, test_file: Optional[str] = None ---------- socrata_token: str My App Token for pulling the NCHS mortality data + backup_dir: str + Directory to which to save raw backup data + custom_run: bool + Flag indicating if the current run is a patch. If so, don't save any data to disk test_file: Optional[str] When not null, name of file from which to read test data @@ -60,6 +72,10 @@ def pull_nchs_mortality_data(socrata_token: str, test_file: Optional[str] = None client = Socrata("data.cdc.gov", socrata_token) results = client.get("r8kw-7aab", limit=10**10) df = pd.DataFrame.from_records(results) + + create_backup_csv(df, backup_dir, custom_run=custom_run, logger=logger) + + if not test_file: # drop "By Total" rows df = df[df["group"].transform(str.lower) == "by week"] diff --git a/nchs_mortality/delphi_nchs_mortality/run.py b/nchs_mortality/delphi_nchs_mortality/run.py index 50ce46cfb..4e88e9d61 100644 --- a/nchs_mortality/delphi_nchs_mortality/run.py +++ b/nchs_mortality/delphi_nchs_mortality/run.py @@ -59,6 +59,8 @@ def run_module(params: Dict[str, Any]): days=date.today().weekday() + 2) export_start_date = export_start_date.strftime('%Y-%m-%d') daily_export_dir = params["common"]["daily_export_dir"] + backup_dir = params["common"]["backup_dir"] + custom_run = params["common"].get("custom_run", False) socrata_token = params["indicator"]["socrata_token"] test_file = params["indicator"].get("test_file", None) @@ -70,7 +72,9 @@ def run_module(params: Dict[str, Any]): daily_arch_diff.update_cache() stats = [] - df_pull = pull_nchs_mortality_data(socrata_token, test_file) + df_pull = pull_nchs_mortality_data( + socrata_token, backup_dir, custom_run=custom_run, test_file=test_file, logger=logger + ) for metric in METRICS: for geo in ["state", "nation"]: if metric == 'percent_of_expected_deaths': diff --git a/nchs_mortality/params.json.template b/nchs_mortality/params.json.template index ed16c620c..2e829de24 100644 --- a/nchs_mortality/params.json.template +++ b/nchs_mortality/params.json.template @@ -2,7 +2,8 @@ "common": { "daily_export_dir": "./daily_receiving", "weekly_export_dir": "./receiving", - "log_filename": "/var/log/indicators/nchs_mortality.log", + "backup_dir": "./raw_data_backups", + "log_filename": "./nchs_mortality.log", "log_exceptions": false }, "indicator": { diff --git a/nchs_mortality/raw_data_backups/.gitignore b/nchs_mortality/raw_data_backups/.gitignore new file mode 100644 index 000000000..552154e09 --- /dev/null +++ b/nchs_mortality/raw_data_backups/.gitignore @@ -0,0 +1,120 @@ +# You should hard commit a prototype for this file, but we +# want to avoid accidental adding of API tokens and other +# private data parameters +params.json + +# Do not commit output files +receiving/*.csv + +# Remove macOS files +.DS_Store + +# virtual environment +dview/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +coverage.xml +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +.static_storage/ +.media/ +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/nchs_mortality/setup.py b/nchs_mortality/setup.py index 3fe354ba4..142db6f35 100644 --- a/nchs_mortality/setup.py +++ b/nchs_mortality/setup.py @@ -2,10 +2,12 @@ from setuptools import find_packages required = [ + "boto3", "darker[isort]~=2.1.1", "delphi-utils", "epiweeks", "freezegun", + "moto~=4.2.14", "numpy", "pandas", "pydocstyle", diff --git a/nchs_mortality/tests/conftest.py b/nchs_mortality/tests/conftest.py index 6ad0f9c59..383d1c782 100644 --- a/nchs_mortality/tests/conftest.py +++ b/nchs_mortality/tests/conftest.py @@ -14,8 +14,10 @@ PARAMS = { "common": { + "custom_run": True, "daily_export_dir": "./daily_receiving", - "weekly_export_dir": "./receiving" + "weekly_export_dir": "./receiving", + "backup_dir": "./raw_data_backups" }, "indicator": { "export_start_date": "2020-04-11", diff --git a/nchs_mortality/tests/raw_data_backups/.gitignore b/nchs_mortality/tests/raw_data_backups/.gitignore new file mode 100644 index 000000000..2b7efbb36 --- /dev/null +++ b/nchs_mortality/tests/raw_data_backups/.gitignore @@ -0,0 +1,2 @@ +*.csv +*.gz \ No newline at end of file diff --git a/nchs_mortality/tests/test_pull.py b/nchs_mortality/tests/test_pull.py index fa58b04a5..4f18210f6 100644 --- a/nchs_mortality/tests/test_pull.py +++ b/nchs_mortality/tests/test_pull.py @@ -1,3 +1,4 @@ +import os import pytest import pandas as pd @@ -34,7 +35,7 @@ def test_standardize_columns(self): pd.testing.assert_frame_equal(expected, df) def test_good_file(self): - df = pull_nchs_mortality_data(SOCRATA_TOKEN, "test_data.csv") + df = pull_nchs_mortality_data(SOCRATA_TOKEN, backup_dir = "", custom_run = True, test_file = "test_data.csv") # Test columns assert ( @@ -90,9 +91,20 @@ def test_good_file(self): def test_bad_file_with_inconsistent_time_col(self): with pytest.raises(ValueError): pull_nchs_mortality_data( - SOCRATA_TOKEN, "bad_data_with_inconsistent_time_col.csv" + SOCRATA_TOKEN, backup_dir = "", custom_run = True, test_file = "bad_data_with_inconsistent_time_col.csv" ) def test_bad_file_with_missing_cols(self): with pytest.raises(ValueError): - pull_nchs_mortality_data(SOCRATA_TOKEN, "bad_data_with_missing_cols.csv") + pull_nchs_mortality_data(SOCRATA_TOKEN, backup_dir = "", custom_run = True, test_file = "bad_data_with_missing_cols.csv") + + def test_backup_today_data(self): + today = pd.Timestamp.today().strftime("%Y%m%d") + backup_dir = "./raw_data_backups" + pull_nchs_mortality_data(SOCRATA_TOKEN, backup_dir = backup_dir, custom_run = False, test_file = "test_data.csv") + backup_file = f"{backup_dir}/{today}.csv.gz" + backup_df = pd.read_csv(backup_file) + source_df = pd.read_csv("test_data/test_data.csv") + pd.testing.assert_frame_equal(source_df, backup_df) + if os.path.exists(backup_file): + os.remove(backup_file) diff --git a/nchs_mortality/version.cfg b/nchs_mortality/version.cfg index f5c28d2cd..ed4b085f1 100644 --- a/nchs_mortality/version.cfg +++ b/nchs_mortality/version.cfg @@ -1 +1 @@ -current_version = 0.3.55 +current_version = 0.3.56 diff --git a/nssp/delphi_nssp/constants.py b/nssp/delphi_nssp/constants.py index ddd2e74b8..9b98d2012 100644 --- a/nssp/delphi_nssp/constants.py +++ b/nssp/delphi_nssp/constants.py @@ -6,6 +6,7 @@ "nation", "state", "county", + "hhs", ] SIGNALS_MAP = { diff --git a/nssp/delphi_nssp/run.py b/nssp/delphi_nssp/run.py index 7c5a3ffac..b22d03c20 100644 --- a/nssp/delphi_nssp/run.py +++ b/nssp/delphi_nssp/run.py @@ -90,7 +90,7 @@ def run_module(params): for geo in GEOS: df = df_pull.copy() df["val"] = df[signal] - logger.info("Generating signal and exporting to CSV", metric=signal) + logger.info("Generating signal and exporting to CSV", geo_type=geo, signal=signal) if geo == "nation": df = df[df["geography"] == "United States"] df["geo_id"] = "us" @@ -111,6 +111,14 @@ def run_module(params): df = geo_mapper.add_geocode(df, "fips", "msa", from_col="fips", new_col="geo_id") df = geo_mapper.aggregate_by_weighted_sum(df, "geo_id", "val", "timestamp", "population") df = df.rename(columns={"weighted_val": "val"}) + elif geo == "hhs": + df = df[(df["county"] == "All") & (df["geography"] != "United States")] + df = df[["geography", "val", "timestamp"]] + df = geo_mapper.add_population_column(df, geocode_type="state_name", geocode_col="geography") + df = geo_mapper.add_geocode(df, "state_name", "state_code", from_col="state_name") + df = geo_mapper.add_geocode(df, "state_code", "hhs", from_col="state_code", new_col="geo_id") + df = geo_mapper.aggregate_by_weighted_sum(df, "geo_id", "val", "timestamp", "population") + df = df.rename(columns={"weighted_val": "val"}) else: df = df[df["county"] != "All"] df["geo_id"] = df["fips"] diff --git a/nwss_wastewater/delphi_nwss/run.py b/nwss_wastewater/delphi_nwss/run.py index 378849ba5..60bfc84c7 100644 --- a/nwss_wastewater/delphi_nwss/run.py +++ b/nwss_wastewater/delphi_nwss/run.py @@ -146,7 +146,7 @@ def run_module(params): df = generate_weights(df, sensor) for geo in GEOS: - logger.info("Generating signal and exporting to CSV", metric=sensor) + logger.info("Generating signal and exporting to CSV", geo_type=geo, signal=sensor) if geo == "nation": agg_df = weighted_nation_sum(df, sensor) else: diff --git a/quidel_covidtest/delphi_quidel_covidtest/pull.py b/quidel_covidtest/delphi_quidel_covidtest/pull.py index d9f23f2ec..560f89456 100644 --- a/quidel_covidtest/delphi_quidel_covidtest/pull.py +++ b/quidel_covidtest/delphi_quidel_covidtest/pull.py @@ -56,7 +56,7 @@ def get_from_s3(start_date, end_date, bucket, logger): seen_files = set() for search_date in [start_date + timedelta(days=x) for x in range(n_days)]: if search_date in s3_files.keys(): - logger.info(f"Pulling data received on {search_date.date()}") + logger.info("Pulling data received on date", search_date=search_date.date()) # Fetch data received on the same day for fn in s3_files[search_date]: @@ -110,11 +110,11 @@ def fix_date(df, logger): df.insert(2, "timestamp", df["TestDate"]) mask = df["TestDate"] <= df["StorageDate"] - logger.info(f"Removing {((len(df) - np.sum(mask)) * 100 / len(df)):.2f}% of unusual data") + logger.info("Removing unusual data", percent=round((len(df) - np.sum(mask)) * 100 / len(df), 2)) df = df[mask] mask = df["StorageDate"] - df["TestDate"] > pd.Timedelta(days=90) - logger.info(f"Fixing {(np.sum(mask) * 100 / len(df)):.2f}% of outdated data") + logger.info("Fixing outdated data", percent=round((np.sum(mask) * 100 / len(df)), 2)) df["timestamp"].values[mask] = df["StorageDate"].values[mask] return df diff --git a/quidel_covidtest/delphi_quidel_covidtest/run.py b/quidel_covidtest/delphi_quidel_covidtest/run.py index a59e0c101..e6974b6aa 100644 --- a/quidel_covidtest/delphi_quidel_covidtest/run.py +++ b/quidel_covidtest/delphi_quidel_covidtest/run.py @@ -164,7 +164,7 @@ def run_module(params: Dict[str, Any]): n_cpu = min(8, cpu_count()) # for parallelization with pool_and_threadedlogger(logger, n_cpu) as (pool, threaded_logger): # for using loggers in multiple threads - logger.info("Parallelizing sensor generation", n_cpu=n_cpu) + logger.info("Parallelizing sensor generation", n_workers=n_cpu) pool_results = [] for geo_res in NONPARENT_GEO_RESOLUTIONS: geo_data, res_key = geo_map(geo_res, data) diff --git a/quidel_covidtest/setup.py b/quidel_covidtest/setup.py index c2791930f..82c80832a 100644 --- a/quidel_covidtest/setup.py +++ b/quidel_covidtest/setup.py @@ -2,6 +2,7 @@ from setuptools import find_packages required = [ + "boto3", "covidcast", "darker[isort]~=2.1.1", "delphi-utils", @@ -14,7 +15,7 @@ "pylint==2.8.3", "pytest-cov", "pytest", - "xlrd==1.2.0", + "xlrd==1.2.0", # needed by Pandas to read Excel files ] setup( diff --git a/quidel_covidtest/version.cfg b/quidel_covidtest/version.cfg index f5c28d2cd..ed4b085f1 100644 --- a/quidel_covidtest/version.cfg +++ b/quidel_covidtest/version.cfg @@ -1 +1 @@ -current_version = 0.3.55 +current_version = 0.3.56 diff --git a/sir_complainsalot/version.cfg b/sir_complainsalot/version.cfg index f5c28d2cd..ed4b085f1 100644 --- a/sir_complainsalot/version.cfg +++ b/sir_complainsalot/version.cfg @@ -1 +1 @@ -current_version = 0.3.55 +current_version = 0.3.56