diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7c34c9165..fa394e9d6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.1.24 +current_version = 4.1.25 commit = False tag = False diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index bfaa3109e..3abbfed0e 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -20,3 +20,5 @@ b9ceb400d9248c8271e8342275664ac5524e335d 07ed83e5768f717ab0f9a62a9209e4e2cffa058d # style(black): format wiki acquisition 923852eafa86b8f8b182d499489249ba8f815843 +# lint: trailing whitespace changes +81179c5f144b8f25421e799e823e18cde43c84f9 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ac24d1e59..efac282d3 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,4 +1,4 @@ -closes|addresses +addresses issue(s) #ISSUE ### Summary: diff --git a/.github/workflows/update_gdocs_data.yml b/.github/workflows/update_gdocs_data.yml index c2d6f0c07..0306b641c 100644 --- a/.github/workflows/update_gdocs_data.yml +++ b/.github/workflows/update_gdocs_data.yml @@ -21,7 +21,10 @@ jobs: restore-keys: | ${{ runner.os }}-pipd- - name: Install Dependencies - run: pip install -r requirements.dev.txt + run: | + pip -V + python -m pip install pip==22.0.2 + pip install -r requirements.dev.txt - name: Update Docs run: inv update-gdoc - name: Create pull request into dev diff --git a/dev/local/setup.cfg b/dev/local/setup.cfg index 4e67b3354..dd30723a4 100644 --- a/dev/local/setup.cfg +++ b/dev/local/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = Delphi Development -version = 4.1.24 +version = 4.1.25 [options] packages = diff --git a/docs/api/covidcast-signals/covid-act-now.md b/docs/api/covidcast-signals/covid-act-now.md index 72a5a9a20..670bda61f 100644 --- a/docs/api/covidcast-signals/covid-act-now.md +++ b/docs/api/covidcast-signals/covid-act-now.md @@ -15,11 +15,13 @@ grand_parent: COVIDcast Main Endpoint * **Time type:** day (see [date format docs](../covidcast_times.md)) * **License:** [CC BY-NC](../covidcast_licensing.md#creative-commons-attribution-noncommercial) -The COVID Act Now (CAN) data source provides COVID-19 testing statistics, such as positivity rates and total tests performed. -The county-level positivity rates and test totals are pulled directly from CAN. -While CAN provides this data potentially from multiple sources, we only use data sourced from the +The [COVID Act Now (CAN)](https://covidactnow.org/) data source provides COVID-19 testing statistics, such as positivity rates and total tests performed. +The county-level positivity rates and test totals are pulled directly from CAN using [their API](https://covidactnow.org/data-api). +While CAN provides this data potentially from multiple sources, we only use data that CAN sources from the [CDC's COVID-19 Integrated County View](https://covid.cdc.gov/covid-data-tracker/#county-view). +Delphi's mirror of the CAN data was deactivated in December 2021 (last issue 2021-12-10) in favor of the [DSEW CPR data](./dsew-cpr.md), which reports the same information under the `covid_naat_pct_positive_7dav` signal. + | Signal | Description | |--------------------------------|----------------------------------------------------------------| @@ -34,9 +36,9 @@ While CAN provides this data potentially from multiple sources, we only use data ## Estimation -The quantities received from CAN / CDC are the county-level positivity rate and total tests, -which are based on the counts of PCR specimens tested. -In particular, they are also already smoothed with a 7-day-average. +We receive county-level positivity rate and total tests from CAN, originating from the CDC. +These quantiles are based on the counts of PCR specimens tested. +They are also already smoothed with a 7-day-average. For a fixed location $$i$$ and time $$t$$, let $$Y_{it}$$ denote the number of PCR specimens tested that have a positive result. Let $$N_{it}$$ denote the total number of PCR specimens tested. @@ -79,38 +81,41 @@ $$ ### Smoothing -No additional smoothing is done to avoid double-smoothing, since the data pulled from CAN / CDC +No additional smoothing is done to avoid double-smoothing, since the CAN data is already smoothed with a 7-day-average. ## Limitations -Estimates for geographical levels beyond counties may be inaccurate due to how aggregations -are done on smoothed values instead of the raw values. Ideally we would aggregate raw values +Estimates for geographical levels beyond counties may be inaccurate because our aggregations +are performed on smoothed values instead of the raw values. +Ideally we would aggregate raw values then smooth, but the raw values are not accessible in this case. -The positivity rate here should not be interpreted as the population positivity rate as +The reported test positivity rate should not be interpreted as the population positivity rate as the testing performed are typically not randomly sampled, especially for early data with lower testing volumes. A few counties, most notably in California, are also not covered by this data source. -Entries with zero total tests performed are also suppressed, even if it was actually the case that +Entries with zero total tests performed are suppressed, even if it was actually the case that no tests were performed for the day. ## Lag and Backfill The lag for these signals varies depending on the reporting patterns of individual counties. Most counties have their latest data report with a lag of 2 days, while others can take 9 days -or more in the case of California counties. +or more, as is the case with California counties. -These signals are also backfilled as backlogged test results could get assigned to older 7-day timeframes. -Most recent test positivity rates do not change substantially with backfill (having a median delta of close to 0). -However, most recent total tests performed is expected to increase in later data revisions (having a median increase of 7%). +Revisions are sometimes made to the data. For example, backlogged test results can get assigned to past dates. +The majority of recent test positivity rates do not change substantially with backfill (having a median delta of close to 0). +However, the majority of recent total tests performed is expected to increase in later data revisions (having a median increase of 7%). Values more than 5 days in the past are expected to remain fairly static (with total tests performed having a median increase of 1% of less), as most major revisions have already occurred. ## Source and Licensing -County-level testing data is scraped by CAN from the +County-level testing data is scraped by [CAN](https://covidactnow.org/) from the [CDC's COVID-19 Integrated County View](https://covid.cdc.gov/covid-data-tracker/#county-view), and made available through [CAN's API](https://covidactnow.org/tools). + +The data is made available under a [CC BY-NC](../covidcast_licensing.md#creative-commons-attribution-noncommercial) license. diff --git a/docs/api/covidcast-signals/hhs.md b/docs/api/covidcast-signals/hhs.md index b6f916e17..a77187532 100644 --- a/docs/api/covidcast-signals/hhs.md +++ b/docs/api/covidcast-signals/hhs.md @@ -1,6 +1,6 @@ --- title: Department of Health & Human Services -parent: Data Sources and Signals +parent: Inactive Signals grand_parent: COVIDcast Main Endpoint --- diff --git a/docs/api/covidcast-signals/indicator-combination b/docs/api/covidcast-signals/indicator-combination.md similarity index 100% rename from docs/api/covidcast-signals/indicator-combination rename to docs/api/covidcast-signals/indicator-combination.md diff --git a/docs/epidata_development.md b/docs/epidata_development.md index 024f1b4dc..eb38e6ae9 100644 --- a/docs/epidata_development.md +++ b/docs/epidata_development.md @@ -49,7 +49,7 @@ $ [sudo] make test pdb=1 $ [sudo] make test test=repos/delphi/delphi-epidata/integrations/acquisition ``` -You can read the commands executed by the Makefile [here](../dev/local/Makefile). +You can read the commands executed by the Makefile [here](https://github.com/cmu-delphi/delphi-epidata/blob/dev/dev/local/Makefile). ## Rapid Iteration and Bind Mounts @@ -87,8 +87,8 @@ You can test your changes manually by: What follows is a worked demonstration based on the `fluview` endpoint. Before starting, make sure that you have the `delphi_database_epidata`, -`delphi_web_epidata`, and `delphi_redis` containers running; if you don't, see -the Makefile instructions above. +`delphi_web_epidata`, and `delphi_redis` containers running (with `docker ps`); +if you don't, see the Makefile instructions above. First, let's insert some fake data into the `fluview` table: diff --git a/docs/symptom-survey/publications.md b/docs/symptom-survey/publications.md index 905be8ef3..3e5e418e6 100644 --- a/docs/symptom-survey/publications.md +++ b/docs/symptom-survey/publications.md @@ -26,6 +26,10 @@ Pandemic"](https://www.pnas.org/topic/548) in *PNAS*: Research publications using the survey data include: +- C.K. Ettman, E. Badillo-Goicoechea, E.A. Stuart (2024). [Financial + strain, schooling modality and mental health of US adults living + with children during the COVID-19 pandemic](https://doi.org/10.1136/jech-2023-221672). + *Journal of Epidemiology & Community Health*. - K. Sasse, R. Mahabir, O. Gkountouna, A. Crooks, A. Croitoru (2024). [Understanding the determinants of vaccine hesitancy in the United States: A comparison of social surveys and social media](https://doi.org/10.1371/journal.pone.0301488). @@ -41,7 +45,7 @@ Research publications using the survey data include: - Z. Yang, R. Krishnan, and B. Li (2024). [The interplay between individual mobility, health risk, and economic choice: A holistic model for COVID-19 policy intervention](https://doi.org/10.1287/ijds.2023.0013). *INFORMS - Journal on Data Science*. + Journal on Data Science* 3 (1), 6-27. - A. Srivastava, J. M. Ramirez, S. Díaz-Aranda, J. Aguilar, A. F. Anta, A. Ortega, and R. E. Lillo (2024). [Nowcasting temporal trends using indirect surveys](https://doi.org/10.1609/aaai.v38i20.30242). In *Proceedings of the 38th AAAI Conference on Artificial Intelligence* 38, diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 3b69eb4cf..e607781ba 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -2,8 +2,8 @@ # standard library import time +import json from json import JSONDecodeError -from requests.models import Response from unittest.mock import MagicMock, patch # first party @@ -306,6 +306,26 @@ def test_sandbox(self, get, post): Epidata.debug = False Epidata.sandbox = False + @patch('requests.get') + def test_version_check(self, get): + """Test that the _version_check() function correctly logs a version discrepancy.""" + class MockJson: + def __init__(self, content, status_code): + self.content = content + self.status_code = status_code + def raise_for_status(self): pass + def json(self): return json.loads(self.content) + get.reset_mock() + get.return_value = MockJson(b'{"info": {"version": "0.0.1"}}', 200) + + Epidata._version_check() + + captured = self.capsys.readouterr() + output = captured.err.splitlines() + self.assertEqual(len(output), 1) + self.assertIn("Client version not up to date", output[0]) + self.assertIn("\'latest_version\': \'0.0.1\'", output[0]) + def test_geo_value(self): """test different variants of geo types: single, *, multi.""" diff --git a/src/acquisition/covid_hosp/common/database.py b/src/acquisition/covid_hosp/common/database.py index efbdb6c45..18c7f377f 100644 --- a/src/acquisition/covid_hosp/common/database.py +++ b/src/acquisition/covid_hosp/common/database.py @@ -11,7 +11,7 @@ # first party import delphi.operations.secrets as secrets -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger Columndef = namedtuple("Columndef", "csv_name sql_name dtype") diff --git a/src/acquisition/covidcast/csv_importer.py b/src/acquisition/covidcast/csv_importer.py index f6122e610..e9893c0da 100644 --- a/src/acquisition/covidcast/csv_importer.py +++ b/src/acquisition/covidcast/csv_importer.py @@ -13,10 +13,9 @@ import pandas as pd # first party -from delphi_utils import Nans +from delphi_utils import get_structured_logger, Nans from delphi.utils.epiweek import delta_epiweeks from delphi.epidata.common.covidcast_row import CovidcastRow -from delphi.epidata.common.logger import get_structured_logger DataFrameRow = NamedTuple('DFRow', [ ('geo_id', str), diff --git a/src/acquisition/covidcast/csv_to_database.py b/src/acquisition/covidcast/csv_to_database.py index be9dad86c..b3642fc51 100644 --- a/src/acquisition/covidcast/csv_to_database.py +++ b/src/acquisition/covidcast/csv_to_database.py @@ -11,7 +11,7 @@ from delphi.epidata.acquisition.covidcast.csv_importer import CsvImporter, PathDetails from delphi.epidata.acquisition.covidcast.database import Database, DBLoadStateException from delphi.epidata.acquisition.covidcast.file_archiver import FileArchiver -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger def get_argument_parser(): diff --git a/src/acquisition/covidcast/database.py b/src/acquisition/covidcast/database.py index 871061b81..5fd56923b 100644 --- a/src/acquisition/covidcast/database.py +++ b/src/acquisition/covidcast/database.py @@ -14,7 +14,7 @@ # first party import delphi.operations.secrets as secrets -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger from delphi.epidata.common.covidcast_row import CovidcastRow @@ -117,16 +117,16 @@ def insert_or_update_batch(self, cc_rows: List[CovidcastRow], batch_size=2**20, get_structured_logger("insert_or_update_batch").fatal(err_msg) raise DBLoadStateException(err_msg) - # NOTE: `value_update_timestamp` is hardcoded to "NOW" (which is appropriate) and + # NOTE: `value_update_timestamp` is hardcoded to "NOW" (which is appropriate) and # `is_latest_issue` is hardcoded to 1 (which is temporary and addressed later in this method) insert_into_loader_sql = f''' INSERT INTO `{self.load_table}` (`source`, `signal`, `time_type`, `geo_type`, `time_value`, `geo_value`, - `value_updated_timestamp`, `value`, `stderr`, `sample_size`, `issue`, `lag`, + `value_updated_timestamp`, `value`, `stderr`, `sample_size`, `issue`, `lag`, `is_latest_issue`, `missing_value`, `missing_stderr`, `missing_sample_size`) VALUES - (%s, %s, %s, %s, %s, %s, - UNIX_TIMESTAMP(NOW()), %s, %s, %s, %s, %s, + (%s, %s, %s, %s, %s, %s, + UNIX_TIMESTAMP(NOW()), %s, %s, %s, %s, %s, 1, %s, %s, %s) ''' @@ -134,11 +134,11 @@ def insert_or_update_batch(self, cc_rows: List[CovidcastRow], batch_size=2**20, # if an entry in the load table is NOT in the latest table, it is clearly now the latest value for that key (so we do nothing (thanks to INNER join)). # if an entry *IS* in both load and latest tables, but latest table issue is newer, unmark is_latest_issue in load. fix_is_latest_issue_sql = f''' - UPDATE - `{self.load_table}` JOIN `{self.latest_view}` - USING (`source`, `signal`, `geo_type`, `geo_value`, `time_type`, `time_value`) - SET `{self.load_table}`.`is_latest_issue`=0 - WHERE `{self.load_table}`.`issue` < `{self.latest_view}`.`issue` + UPDATE + `{self.load_table}` JOIN `{self.latest_view}` + USING (`source`, `signal`, `geo_type`, `geo_value`, `time_type`, `time_value`) + SET `{self.load_table}`.`is_latest_issue`=0 + WHERE `{self.load_table}`.`issue` < `{self.latest_view}`.`issue` ''' # TODO: consider handling cc_rows as a generator instead of a list diff --git a/src/acquisition/covidcast/file_archiver.py b/src/acquisition/covidcast/file_archiver.py index 802590871..07bd453f9 100644 --- a/src/acquisition/covidcast/file_archiver.py +++ b/src/acquisition/covidcast/file_archiver.py @@ -6,7 +6,7 @@ import shutil # first party -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger class FileArchiver: """Archives files by moving and compressing.""" diff --git a/src/client/delphi_epidata.R b/src/client/delphi_epidata.R index 96e168e48..fd461de00 100644 --- a/src/client/delphi_epidata.R +++ b/src/client/delphi_epidata.R @@ -15,7 +15,7 @@ Epidata <- (function() { # API base url BASE_URL <- getOption('epidata.url', default = 'https://api.delphi.cmu.edu/epidata/') - client_version <- '4.1.24' + client_version <- '4.1.25' auth <- getOption("epidata.auth", default = NA) diff --git a/src/client/delphi_epidata.js b/src/client/delphi_epidata.js index 375c3ba01..7afa235c0 100644 --- a/src/client/delphi_epidata.js +++ b/src/client/delphi_epidata.js @@ -22,7 +22,7 @@ } })(this, function (exports, fetchImpl, jQuery) { const BASE_URL = "https://api.delphi.cmu.edu/epidata/"; - const client_version = "4.1.24"; + const client_version = "4.1.25"; // Helper function to cast values and/or ranges to strings function _listitem(value) { diff --git a/src/client/delphi_epidata.py b/src/client/delphi_epidata.py index 9c55e0b73..998c85281 100644 --- a/src/client/delphi_epidata.py +++ b/src/client/delphi_epidata.py @@ -18,7 +18,7 @@ from aiohttp import ClientSession, TCPConnector, BasicAuth -__version__ = "4.1.24" +__version__ = "4.1.25" _HEADERS = {"user-agent": "delphi_epidata/" + __version__ + " (Python)"} @@ -43,8 +43,6 @@ class Epidata: BASE_URL = "https://api.delphi.cmu.edu/epidata" auth = None - client_version = __version__ - debug = False # if True, prints extra logging statements sandbox = False # if True, will not execute any queries @@ -54,6 +52,25 @@ def log(evt, **kwargs): kwargs['timestamp'] = time.strftime("%Y-%m-%d %H:%M:%S %z") return sys.stderr.write(str(kwargs) + "\n") + # Check that this client's version matches the most recent available. This + # is intended to run just once per program execution, on initial module load. + # See the bottom of this file for the ultimate call to this method. + @staticmethod + def _version_check(): + try: + request = requests.get('https://pypi.org/pypi/delphi-epidata/json', timeout=5) + latest_version = request.json()['info']['version'] + except Exception as e: + Epidata.log("Error getting latest client version", exception=str(e)) + return + + if latest_version != __version__: + Epidata.log( + "Client version not up to date", + client_version=__version__, + latest_version=latest_version + ) + # Helper function to cast values and/or ranges to strings @staticmethod def _listitem(value): @@ -692,3 +709,10 @@ async def async_make_calls(param_combos): future = asyncio.ensure_future(async_make_calls(param_list)) responses = loop.run_until_complete(future) return responses + + + +# This should only run once per program execution, on initial module load, +# as a result of how Python's module system works: +# https://docs.python.org/3/reference/import.html#the-module-cache +Epidata._version_check() diff --git a/src/client/packaging/npm/package.json b/src/client/packaging/npm/package.json index 44468d7f7..c88d0c6ec 100644 --- a/src/client/packaging/npm/package.json +++ b/src/client/packaging/npm/package.json @@ -2,7 +2,7 @@ "name": "delphi_epidata", "description": "Delphi Epidata API Client", "authors": "Delphi Group", - "version": "4.1.24", + "version": "4.1.25", "license": "MIT", "homepage": "https://github.com/cmu-delphi/delphi-epidata", "bugs": { diff --git a/src/client/packaging/pypi/.bumpversion.cfg b/src/client/packaging/pypi/.bumpversion.cfg index 5413f5eca..643580e4b 100644 --- a/src/client/packaging/pypi/.bumpversion.cfg +++ b/src/client/packaging/pypi/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.1.24 +current_version = 4.1.25 commit = False tag = False diff --git a/src/client/packaging/pypi/CHANGELOG.md b/src/client/packaging/pypi/CHANGELOG.md index e053619c8..0baf220c8 100644 --- a/src/client/packaging/pypi/CHANGELOG.md +++ b/src/client/packaging/pypi/CHANGELOG.md @@ -3,6 +3,15 @@ All notable future changes to the `delphi_epidata` python client will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/). +## [4.1.25] - 2024-07-29 + +### Includes +- https://github.com/cmu-delphi/delphi-epidata/pull/1456 +- https://github.com/cmu-delphi/delphi-epidata/pull/1497 + +### Changed +- Added a one-time check which logs a warning when the newest client version does not match the client version in use. + ## [4.1.24] - 2024-07-09 ### Includes diff --git a/src/client/packaging/pypi/pyproject.toml b/src/client/packaging/pypi/pyproject.toml index 7acf8dea7..d869f42e4 100644 --- a/src/client/packaging/pypi/pyproject.toml +++ b/src/client/packaging/pypi/pyproject.toml @@ -18,7 +18,7 @@ build-backend = "setuptools.build_meta" # If not defined, then legacy behavior c [project] name = "delphi_epidata" # REQUIRED, is the only field that cannot be marked as dynamic. -version = "4.1.24" +version = "4.1.25" description = "A programmatic interface to Delphi's Epidata API." readme = "README.md" license = { file = "LICENSE" } diff --git a/src/common/logger.py b/src/common/logger.py deleted file mode 100644 index d04ff7673..000000000 --- a/src/common/logger.py +++ /dev/null @@ -1,254 +0,0 @@ -"""Structured logger utility for creating JSON logs.""" - -# the Delphi group uses two ~identical versions of this file. -# try to keep them in sync with edits, for sanity. -# https://github.com/cmu-delphi/covidcast-indicators/blob/main/_delphi_utils_python/delphi_utils/logger.py # pylint: disable=line-too-long -# https://github.com/cmu-delphi/delphi-epidata/blob/dev/src/common/logger.py - -import contextlib -import logging -import multiprocessing -import os -import sys -import threading -from traceback import format_exception - -import structlog - - -def handle_exceptions(logger): - """Handle exceptions using the provided logger.""" - - def exception_handler(scope, etype, value, traceback): - logger.exception("Top-level exception occurred", - scope=scope, exc_info=(etype, value, traceback)) - - def sys_exception_handler(etype, value, traceback): - exception_handler("sys", etype, value, traceback) - - def threading_exception_handler(args): - if args.exc_type == SystemExit and args.exc_value.code == 0: - # `sys.exit(0)` is considered "successful termination": - # https://docs.python.org/3/library/sys.html#sys.exit - logger.debug("normal thread exit", thread=args.thread, - stack="".join( - format_exception( - args.exc_type, args.exc_value, args.exc_traceback))) - else: - exception_handler(f"thread: {args.thread}", - args.exc_type, args.exc_value, args.exc_traceback) - - sys.excepthook = sys_exception_handler - threading.excepthook = threading_exception_handler - - -def get_structured_logger(name=__name__, - filename=None, - log_exceptions=True): - """Create a new structlog logger. - - Use the logger returned from this in indicator code using the standard - wrapper calls, e.g.: - - logger = get_structured_logger(__name__) - logger.warning("Error", type="Signal too low"). - - The output will be rendered as JSON which can easily be consumed by logs - processors. - - See the structlog documentation for details. - - Parameters - --------- - name: Name to use for logger (included in log lines), __name__ from caller - is a good choice. - filename: An (optional) file to write log output. - """ - # Set the underlying logging configuration - if "LOG_DEBUG" in os.environ: - log_level = logging.DEBUG - else: - log_level = logging.INFO - - logging.basicConfig( - format="%(message)s", - level=log_level, - handlers=[logging.StreamHandler()]) - - def add_pid(_logger, _method_name, event_dict): - """Add current PID to the event dict.""" - event_dict["pid"] = os.getpid() - return event_dict - - # Configure structlog. This uses many of the standard suggestions from - # the structlog documentation. - structlog.configure( - processors=[ - # Filter out log levels we are not tracking. - structlog.stdlib.filter_by_level, - # Include logger name in output. - structlog.stdlib.add_logger_name, - # Include log level in output. - structlog.stdlib.add_log_level, - # Include PID in output. - add_pid, - # Allow formatting into arguments e.g., logger.info("Hello, %s", - # name) - structlog.stdlib.PositionalArgumentsFormatter(), - # Add timestamps. - structlog.processors.TimeStamper(fmt="iso"), - # Match support for exception logging in the standard logger. - structlog.processors.StackInfoRenderer(), - structlog.processors.format_exc_info, - # Decode unicode characters - structlog.processors.UnicodeDecoder(), - # Render as JSON - structlog.processors.JSONRenderer(), - ], - # Use a dict class for keeping track of data. - context_class=dict, - # Use a standard logger for the actual log call. - logger_factory=structlog.stdlib.LoggerFactory(), - # Use a standard wrapper class for utilities like log.warning() - wrapper_class=structlog.stdlib.BoundLogger, - # Cache the logger - cache_logger_on_first_use=True, - ) - - # Create the underlying python logger and wrap it with structlog - system_logger = logging.getLogger(name) - if filename and not system_logger.handlers: - system_logger.addHandler(logging.FileHandler(filename)) - system_logger.setLevel(log_level) - logger = structlog.wrap_logger(system_logger) - - if log_exceptions: - handle_exceptions(logger) - - return logger - - -class LoggerThread(): - """ - A construct to use a logger from multiprocessing workers/jobs. - - the bare structlog loggers are thread-safe but not multiprocessing-safe. - a `LoggerThread` will spawn a thread that listens to a mp.Queue - and logs messages from it with the provided logger, - so other processes can send logging messages to it - via the logger-like `SubLogger` interface. - the SubLogger even logs the pid of the caller. - - this is good to use with a set of jobs that are part of a mp.Pool, - but isnt recommended for general use - because of overhead from threading and multiprocessing, - and because it might introduce lag to log messages. - - somewhat inspired by: - docs.python.org/3/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes - """ - - class SubLogger(): - """MP-safe logger-like interface to convey log messages to a listening LoggerThread.""" - - def __init__(self, queue): - """Create SubLogger with a bound queue.""" - self.queue = queue - - def _log(self, level, *args, **kwargs): - kwargs_plus = {'sub_pid': multiprocessing.current_process().pid} - kwargs_plus.update(kwargs) - self.queue.put([level, args, kwargs_plus]) - - def debug(self, *args, **kwargs): - """Log a DEBUG level message.""" - self._log(logging.DEBUG, *args, **kwargs) - - def info(self, *args, **kwargs): - """Log an INFO level message.""" - self._log(logging.INFO, *args, **kwargs) - - def warning(self, *args, **kwargs): - """Log a WARNING level message.""" - self._log(logging.WARNING, *args, **kwargs) - - def error(self, *args, **kwargs): - """Log an ERROR level message.""" - self._log(logging.ERROR, *args, **kwargs) - - def critical(self, *args, **kwargs): - """Log a CRITICAL level message.""" - self._log(logging.CRITICAL, *args, **kwargs) - - - def get_sublogger(self): - """Retrieve SubLogger for this LoggerThread.""" - return self.sublogger - - def __init__(self, logger, q=None): - """Create and start LoggerThread with supplied logger, creating a queue if not provided.""" - self.logger = logger - if q: - self.msg_queue = q - else: - self.msg_queue = multiprocessing.Queue() - - def logger_thread_worker(): - logger.info('thread started') - while True: - msg = self.msg_queue.get() - if msg == 'STOP': - logger.debug('received stop signal') - break - level, args, kwargs = msg - if level in [logging.DEBUG, logging.INFO, logging.WARNING, - logging.ERROR, logging.CRITICAL]: - logger.log(level, *args, **kwargs) - else: - logger.error('received unknown logging level! exiting...', - level=level, args_kwargs=(args, kwargs)) - break - logger.debug('stopping thread') - - self.thread = threading.Thread(target=logger_thread_worker, - name="LoggerThread__"+logger.name) - logger.debug('starting thread') - self.thread.start() - - self.sublogger = LoggerThread.SubLogger(self.msg_queue) - self.running = True - - def stop(self): - """Terminate this LoggerThread.""" - if not self.running: - self.logger.warning('thread already stopped') - return - self.logger.debug('sending stop signal') - self.msg_queue.put('STOP') - self.thread.join() - self.running = False - self.logger.info('thread stopped') - - -@contextlib.contextmanager -def pool_and_threadedlogger(logger, *poolargs): - """ - Provide (to a context) a multiprocessing Pool and a proxy to the supplied logger. - - Emulates the multiprocessing.Pool() context manager, - but also provides (via a LoggerThread) a SubLogger proxy to logger - that can be safely used by pool workers. - The SubLogger proxy interface supports these methods: debug, info, warning, error, - and critical. - Also "cleans up" the pool by waiting for workers to complete - as it exits the context. - """ - with multiprocessing.Manager() as manager: - logger_thread = LoggerThread(logger, manager.Queue()) - try: - with multiprocessing.Pool(*poolargs) as pool: - yield pool, logger_thread.get_sublogger() - pool.close() - pool.join() - finally: - logger_thread.stop() diff --git a/src/maintenance/covidcast_meta_cache_updater.py b/src/maintenance/covidcast_meta_cache_updater.py index c5f7fe3e8..cb0b2703f 100644 --- a/src/maintenance/covidcast_meta_cache_updater.py +++ b/src/maintenance/covidcast_meta_cache_updater.py @@ -7,7 +7,7 @@ # first party from delphi.epidata.acquisition.covidcast.database import Database -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger from delphi.epidata.client.delphi_epidata import Epidata def get_argument_parser(): diff --git a/src/maintenance/delete_batch.py b/src/maintenance/delete_batch.py index 31a25ef2a..8e8298817 100644 --- a/src/maintenance/delete_batch.py +++ b/src/maintenance/delete_batch.py @@ -8,7 +8,7 @@ # first party from delphi.epidata.acquisition.covidcast.database import Database -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger def get_argument_parser(): diff --git a/src/maintenance/signal_dash_data_generator.py b/src/maintenance/signal_dash_data_generator.py index b7f1048f5..5a7067f83 100644 --- a/src/maintenance/signal_dash_data_generator.py +++ b/src/maintenance/signal_dash_data_generator.py @@ -15,7 +15,7 @@ # first party import covidcast import delphi.operations.secrets as secrets -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger LOOKBACK_DAYS_FOR_COVERAGE = 56 @@ -150,11 +150,11 @@ def write_coverage( def get_enabled_signals(self) -> List[DashboardSignal]: """Retrieve all enabled signals from the database""" - select_statement = f'''SELECT `id`, + select_statement = f'''SELECT `id`, `name`, `source`, `covidcast_signal`, - `latest_coverage_update`, + `latest_coverage_update`, `latest_status_update` FROM `{Database.SIGNAL_TABLE_NAME}` WHERE `enabled` @@ -208,7 +208,7 @@ def get_coverage(dashboard_signal: DashboardSignal) -> List[DashboardSignalCover lambda x: pd.to_datetime(Week(x // 100, x % 100).startdate())) signal_coverage_list = [] - + for _, row in count_by_geo_type_df.iterrows(): signal_coverage = DashboardSignalCoverage( signal_id=dashboard_signal.db_id, diff --git a/src/server/_common.py b/src/server/_common.py index 33a3f9c48..692b83491 100644 --- a/src/server/_common.py +++ b/src/server/_common.py @@ -7,7 +7,7 @@ from werkzeug.exceptions import Unauthorized from werkzeug.local import LocalProxy -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger from ._config import SECRET, REVERSE_PROXY_DEPTH from ._db import engine from ._exceptions import DatabaseErrorException, EpiDataException diff --git a/src/server/_config.py b/src/server/_config.py index 9ef373d1e..7ca9ae486 100644 --- a/src/server/_config.py +++ b/src/server/_config.py @@ -7,7 +7,7 @@ load_dotenv() -VERSION = "4.1.24" +VERSION = "4.1.25" MAX_RESULTS = int(10e6) MAX_COMPATIBILITY_RESULTS = int(3650) diff --git a/src/server/_printer.py b/src/server/_printer.py index 5616787a2..6df6d62b9 100644 --- a/src/server/_printer.py +++ b/src/server/_printer.py @@ -8,7 +8,7 @@ from ._config import MAX_RESULTS, MAX_COMPATIBILITY_RESULTS from ._common import is_compatibility_mode, log_info_with_request -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger def print_non_standard(format: str, data): diff --git a/src/server/_security.py b/src/server/_security.py index c47f948a5..2e127debf 100644 --- a/src/server/_security.py +++ b/src/server/_security.py @@ -3,7 +3,7 @@ from typing import Optional, cast import redis -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger from flask import g, request from werkzeug.exceptions import Unauthorized from werkzeug.local import LocalProxy diff --git a/src/server/admin/models.py b/src/server/admin/models.py index f5c0d54ed..e0ef86b0f 100644 --- a/src/server/admin/models.py +++ b/src/server/admin/models.py @@ -4,7 +4,7 @@ from copy import deepcopy from .._db import Session, WriteSession, default_session -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger from typing import Set, Optional, List from datetime import datetime as dtime diff --git a/src/server/endpoints/covidcast.py b/src/server/endpoints/covidcast.py index 11de3cbca..3d7d99e82 100644 --- a/src/server/endpoints/covidcast.py +++ b/src/server/endpoints/covidcast.py @@ -36,7 +36,7 @@ from .covidcast_utils import compute_trend, compute_trends, compute_trend_value, CovidcastMetaEntry from ..utils import shift_day_value, day_to_time_value, time_value_to_iso, time_value_to_day, shift_week_value, time_value_to_week, guess_time_value_is_day, week_to_time_value, TimeValues from .covidcast_utils.model import TimeType, count_signal_time_types, data_sources, create_source_signal_alias_mapper -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger # first argument is the endpoint name bp = Blueprint("covidcast", __name__) diff --git a/src/server/endpoints/covidcast_meta.py b/src/server/endpoints/covidcast_meta.py index 35dc9f12e..8c2219ae7 100644 --- a/src/server/endpoints/covidcast_meta.py +++ b/src/server/endpoints/covidcast_meta.py @@ -9,7 +9,7 @@ from .._printer import create_printer from .._query import filter_fields from .._security import current_user, sources_protected_by_roles -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger bp = Blueprint("covidcast_meta", __name__) diff --git a/src/server/endpoints/covidcast_utils/db_signals.csv b/src/server/endpoints/covidcast_utils/db_signals.csv index bcddbc442..b14d477ae 100644 --- a/src/server/endpoints/covidcast_utils/db_signals.csv +++ b/src/server/endpoints/covidcast_utils/db_signals.csv @@ -1560,4 +1560,60 @@ usa-facts,deaths_cumulative_num,TRUE,deaths_7dav_incidence_num,TRUE,"Confirmed C usa-facts,deaths_cumulative_num,TRUE,deaths_7dav_incidence_prop,FALSE,"Confirmed COVID Deaths (Daily new, 7-day average, per 100k people)",FALSE,"Daily new confirmed COVID deaths, 7-day average, per 100k people",,USAFacts,covid,Deaths,USA,"county,hrr,msa,state,hhs,nation","hrr,msa,state,hhs,nation",2020-02-01,NA,2023-01-02,NA,day,Date,weekly,2-8 days,"None. The raw data reports cumulative cases and deaths, which Delphi diffs to compute incidence. Raw cumulative figures are sometimes corrected by adjusting the reported value for a single day, but revisions do not affect past report dates.",All,None,dead,None,Data is available for all counties and some territorial county equivalents. Data is available for all states.,Value,per100k,late,bad,TRUE,FALSE,FALSE,FALSE,FALSE,public,public,CC BY,See license,NA, usa-facts,deaths_cumulative_num,TRUE,deaths_cumulative_prop,FALSE,"Confirmed COVID Deaths (Cumulative, per 100k people)",FALSE,"Cumulative confirmed COVID deaths, per 100k people",,USAFacts,covid,Deaths,USA,"county,hrr,msa,state,hhs,nation","hrr,msa,state,hhs,nation",2020-01-25,NA,2023-01-02,NA,day,Date,weekly,2-8 days,"None. The raw data reports cumulative cases and deaths, which Delphi diffs to compute incidence. Raw cumulative figures are sometimes corrected by adjusting the reported value for a single day, but revisions do not affect past report dates.",All,None,dead,None,Data is available for all counties and some territorial county equivalents. Data is available for all states.,Value,per100k,late,bad,FALSE,FALSE,TRUE,FALSE,FALSE,public,public,CC BY,See license,NA, usa-facts,deaths_cumulative_num,TRUE,deaths_incidence_num,TRUE,Confirmed COVID Deaths (Daily new),FALSE,Daily new confirmed COVID deaths,,USAFacts,covid,Deaths,USA,"county,hrr,msa,state,hhs,nation","hrr,msa,state,hhs,nation",2020-01-25,NA,2023-01-02,NA,day,Date,weekly,2-8 days,"None. The raw data reports cumulative cases and deaths, which Delphi diffs to compute incidence. Raw cumulative figures are sometimes corrected by adjusting the reported value for a single day, but revisions do not affect past report dates.",All,None,dead,None,Data is available for all counties and some territorial county equivalents. Data is available for all states.,Value,count,late,bad,FALSE,FALSE,FALSE,FALSE,FALSE,public,public,CC BY,See license,NA, -usa-facts,deaths_cumulative_num,TRUE,deaths_incidence_prop,FALSE,"Confirmed COVID Deaths (Daily new, per 100k people)",FALSE,"Daily new confirmed COVID deaths, per 100k people",,USAFacts,covid,Deaths,USA,"county,hrr,msa,state,hhs,nation","hrr,msa,state,hhs,nation",2020-01-25,NA,2023-01-02,NA,day,Date,weekly,2-8 days,"None. The raw data reports cumulative cases and deaths, which Delphi diffs to compute incidence. Raw cumulative figures are sometimes corrected by adjusting the reported value for a single day, but revisions do not affect past report dates.",All,None,dead,None,Data is available for all counties and some territorial county equivalents. Data is available for all states.,Value,per100k,late,bad,FALSE,FALSE,FALSE,FALSE,FALSE,public,public,CC BY,See license,NA, \ No newline at end of file +usa-facts,deaths_cumulative_num,TRUE,deaths_incidence_prop,FALSE,"Confirmed COVID Deaths (Daily new, per 100k people)",FALSE,"Daily new confirmed COVID deaths, per 100k people",,USAFacts,covid,Deaths,USA,"county,hrr,msa,state,hhs,nation","hrr,msa,state,hhs,nation",2020-01-25,NA,2023-01-02,NA,day,Date,weekly,2-8 days,"None. The raw data reports cumulative cases and deaths, which Delphi diffs to compute incidence. Raw cumulative figures are sometimes corrected by adjusting the reported value for a single day, but revisions do not affect past report dates.",All,None,dead,None,Data is available for all counties and some territorial county equivalents. Data is available for all states.,Value,per100k,late,bad,FALSE,FALSE,FALSE,FALSE,FALSE,public,public,CC BY,See license,NA, +nssp,pct_ed_visits_covid,FALSE,pct_ed_visits_covid,FALSE,COVID Emergency Department Visits (Percent of total ED visits),TRUE,Percent of ED visits that had a discharge diagnosis code of COVID-19,Percent of ED visits that had a discharge diagnosis code of COVID-19,National Syndromic Surveillance Program,covid,Hospitalizations,USA,"county,state,hrr,msa","hrr,msa",2022-10-01,,ongoing,,week,Week,weekly,,,All,None,hospitalized,,"Data is available for 78% of US emergency departments. California, Colorado, Missouri, Oklahoma, and Virginia have the most noticeable gaps in coverage, with many counties in those states having either no eligible EDs or having no recently reported data in NSSP. However, most states have some counties that do not contain any reporting EDs. + +NSSP does not report county-level data for all counties with reporting EDs; some reporting EDs are only included in state-level values. + +The following states report no data through NSSP at the county level: CA, WA, AK, AZ, AL, CO, SD, ND, MO, AR, FL, OH, NH, CT, NJ. + +South Dakota, Missouri, and territories report no data through NSSP at the state level.",Percentage,percent,other,bad,FALSE,FALSE,FALSE,FALSE,FALSE,,,,,, +nssp,pct_ed_visits_influenza,FALSE,pct_ed_visits_influenza,FALSE,Influenza Emergency Department Visits (Percent of total ED visits),TRUE,Percent of ED visits that had a discharge diagnosis code of influenza,Percent of ED visits that had a discharge diagnosis code of influenza,National Syndromic Surveillance Program,flu,Hospitalizations,USA,"county,state,hrr,msa","hrr,msa",2022-10-01,,ongoing,,week,Week,weekly,,,All,None,hospitalized,,"Data is available for 78% of US emergency departments. California, Colorado, Missouri, Oklahoma, and Virginia have the most noticeable gaps in coverage, with many counties in those states having either no eligible EDs or having no recently reported data in NSSP. However, most states have some counties that do not contain any reporting EDs. + +NSSP does not report county-level data for all counties with reporting EDs; some reporting EDs are only included in state-level values. + +The following states report no data through NSSP at the county level: CA, WA, AK, AZ, AL, CO, SD, ND, MO, AR, FL, OH, NH, CT, NJ. + +South Dakota, Missouri, and territories report no data through NSSP at the state level.",Percentage,percent,other,bad,FALSE,FALSE,FALSE,FALSE,FALSE,,,,,, +nssp,pct_ed_visits_rsv,FALSE,pct_ed_visits_rsv,FALSE,COVID Emergency Department Visits (Percent of total ED visits),TRUE,Percent of ED visits that had a discharge diagnosis code of rsv,Percent of ED visits that had a discharge diagnosis code of rsv,National Syndromic Surveillance Program,rsv,Hospitalizations,USA,"county,state,hrr,msa","hrr,msa",2022-10-01,,ongoing,,week,Week,weekly,,,All,None,hospitalized,,"Data is available for 78% of US emergency departments. California, Colorado, Missouri, Oklahoma, and Virginia have the most noticeable gaps in coverage, with many counties in those states having either no eligible EDs or having no recently reported data in NSSP. However, most states have some counties that do not contain any reporting EDs. + +NSSP does not report county-level data for all counties with reporting EDs; some reporting EDs are only included in state-level values. + +The following states report no data through NSSP at the county level: CA, WA, AK, AZ, AL, CO, SD, ND, MO, AR, FL, OH, NH, CT, NJ. + +South Dakota, Missouri, and territories report no data through NSSP at the state level.",Percentage,percent,other,bad,FALSE,FALSE,FALSE,FALSE,FALSE,,,,,, +nssp,pct_ed_visits_combined,FALSE,pct_ed_visits_combined,FALSE,"Emergency Department Visits for COVID, Influenza, and RSV Combined (Percent of total ED visits)",TRUE,"Percent of ED visits that had a discharge diagnosis code of COVID-19, influenza, or rsv","Percent of ED visits that had a discharge diagnosis code of COVID-19, influenza, or rsv",National Syndromic Surveillance Program,"covid, flu, rsv",Hospitalizations,USA,"county,state,hrr,msa","hrr,msa",2022-10-01,,ongoing,,week,Week,weekly,,,All,None,hospitalized,,"Data is available for 78% of US emergency departments. California, Colorado, Missouri, Oklahoma, and Virginia have the most noticeable gaps in coverage, with many counties in those states having either no eligible EDs or having no recently reported data in NSSP. However, most states have some counties that do not contain any reporting EDs. + +NSSP does not report county-level data for all counties with reporting EDs; some reporting EDs are only included in state-level values. + +The following states report no data through NSSP at the county level: CA, WA, AK, AZ, AL, CO, SD, ND, MO, AR, FL, OH, NH, CT, NJ. + +South Dakota, Missouri, and territories report no data through NSSP at the state level.",Percentage,percent,other,bad,FALSE,FALSE,FALSE,FALSE,FALSE,,,,,, +nssp,pct_ed_visits_covid,TRUE,smoothed_pct_ed_visits_covid,FALSE,"COVID Emergency Department Visits (Percent of total ED visits, 3-week average)",TRUE,3-week moving average of percent of ED visits that had a discharge diagnosis code of COVID-19,3-week moving average of percent of ED visits that had a discharge diagnosis code of COVID-19,National Syndromic Surveillance Program,covid,Hospitalizations,USA,"county,state,hrr,msa","hrr,msa",2022-10-01,,ongoing,,week,Week,weekly,,,All,None,hospitalized,,"Data is available for 78% of US emergency departments. California, Colorado, Missouri, Oklahoma, and Virginia have the most noticeable gaps in coverage, with many counties in those states having either no eligible EDs or having no recently reported data in NSSP. However, most states have some counties that do not contain any reporting EDs. + +NSSP does not report county-level data for all counties with reporting EDs; some reporting EDs are only included in state-level values. + +The following states report no data through NSSP at the county level: CA, WA, AK, AZ, AL, CO, SD, ND, MO, AR, FL, OH, NH, CT, NJ. + +South Dakota, Missouri, and territories report no data through NSSP at the state level.",Percentage,percent,other,bad,TRUE,FALSE,FALSE,FALSE,FALSE,,,,,, +nssp,pct_ed_visits_influenza,TRUE,smoothed_pct_ed_visits_influenza,FALSE,Influenza Emergency Department Visits (Percent of total ED visits),TRUE,3-week moving average of percent of ED visits that had a discharge diagnosis code of influenza,3-week moving average of percent of ED visits that had a discharge diagnosis code of influenza,National Syndromic Surveillance Program,flu,Hospitalizations,USA,"county,state,hrr,msa","hrr,msa",2022-10-01,,ongoing,,week,Week,weekly,,,All,None,hospitalized,,"Data is available for 78% of US emergency departments. California, Colorado, Missouri, Oklahoma, and Virginia have the most noticeable gaps in coverage, with many counties in those states having either no eligible EDs or having no recently reported data in NSSP. However, most states have some counties that do not contain any reporting EDs. + +NSSP does not report county-level data for all counties with reporting EDs; some reporting EDs are only included in state-level values. + +The following states report no data through NSSP at the county level: CA, WA, AK, AZ, AL, CO, SD, ND, MO, AR, FL, OH, NH, CT, NJ. + +South Dakota, Missouri, and territories report no data through NSSP at the state level.",Percentage,percent,other,bad,TRUE,FALSE,FALSE,FALSE,FALSE,,,,,, +nssp,pct_ed_visits_rsv,TRUE,smoothed_pct_ed_visits_rsv,FALSE,COVID Emergency Department Visits (Percent of total ED visits),TRUE,3-week moving average of percent of ED visits that had a discharge diagnosis code of rsv,3-week moving average of percent of ED visits that had a discharge diagnosis code of rsv,National Syndromic Surveillance Program,rsv,Hospitalizations,USA,"county,state,hrr,msa","hrr,msa",2022-10-01,,ongoing,,week,Week,weekly,,,All,None,hospitalized,,"Data is available for 78% of US emergency departments. California, Colorado, Missouri, Oklahoma, and Virginia have the most noticeable gaps in coverage, with many counties in those states having either no eligible EDs or having no recently reported data in NSSP. However, most states have some counties that do not contain any reporting EDs. + +NSSP does not report county-level data for all counties with reporting EDs; some reporting EDs are only included in state-level values. + +The following states report no data through NSSP at the county level: CA, WA, AK, AZ, AL, CO, SD, ND, MO, AR, FL, OH, NH, CT, NJ. + +South Dakota, Missouri, and territories report no data through NSSP at the state level.",Percentage,percent,other,bad,TRUE,FALSE,FALSE,FALSE,FALSE,,,,,, +nssp,pct_ed_visits_combined,TRUE,smoothed_pct_ed_visits_combined,FALSE,"Emergency Department Visits for COVID, Influenza, and RSV Combined (Percent of total ED visits)",TRUE,"3-week moving average of percent of ED visits that had a discharge diagnosis code of COVID-19, influenza, or rsv","3-week moving average of percent of ED visits that had a discharge diagnosis code of COVID-19, influenza, or rsv",National Syndromic Surveillance Program,"covid, flu, rsv",Hospitalizations,USA,"county,state,hrr,msa","hrr,msa",2022-10-01,,ongoing,,week,Week,weekly,,,All,None,hospitalized,,"Data is available for 78% of US emergency departments. California, Colorado, Missouri, Oklahoma, and Virginia have the most noticeable gaps in coverage, with many counties in those states having either no eligible EDs or having no recently reported data in NSSP. However, most states have some counties that do not contain any reporting EDs. + +NSSP does not report county-level data for all counties with reporting EDs; some reporting EDs are only included in state-level values. + +The following states report no data through NSSP at the county level: CA, WA, AK, AZ, AL, CO, SD, ND, MO, AR, FL, OH, NH, CT, NJ. + +South Dakota, Missouri, and territories report no data through NSSP at the state level.",Percentage,percent,other,bad,TRUE,FALSE,FALSE,FALSE,FALSE,,,,,, \ No newline at end of file diff --git a/src/server/endpoints/covidcast_utils/db_sources.csv b/src/server/endpoints/covidcast_utils/db_sources.csv index 0a8349131..9b9548551 100644 --- a/src/server/endpoints/covidcast_utils/db_sources.csv +++ b/src/server/endpoints/covidcast_utils/db_sources.csv @@ -1,26 +1,27 @@ -DB Source,Source Subdivision,Name,Description,Reference Signal,License,DUA,Link -chng,chng,Change Healthcare,Change Healthcare is a healthcare technology company that aggregates medical claims data from many healthcare providers. This source includes aggregated counts of claims with confirmed COVID-19 or COVID-related symptoms. All claims data has been de-identified in accordance with HIPAA privacy regulations. ,smoothed_outpatient_cli,CC BY-NC,https://cmu.box.com/s/cto4to822zecr3oyq1kkk9xmzhtq9tl2,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/chng.html) -covid-act-now,covid-act-now,Covid Act Now (CAN),"COVID Act Now (CAN) tracks COVID-19 testing statistics, such as positivity rates and total tests performed. This source only includes CAN data from the CDC's COVID-19 Integrated County View.",pcr_specimen_total_tests,CC BY-NC,,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/covid-act-now.html) -doctor-visits,doctor-visits,Doctor Visits From Claims,"Information about outpatient visits, provided to us by health system partners. Using outpatient claims counts, we estimate the percentage of COVID-related doctor's visits in a given location, on a given day.",smoothed_cli,CC BY,https://cmu.box.com/s/l2tz6kmiws6jyty2azwb43poiepz0565,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/doctor-visits.html) -fb-survey,fb-survey,Delphi US COVID-19 Trends and Impact Survey,"We conduct the Delphi US COVID-19 Trends and Impact Survey (CTIS) in partnership with Facebook. Every day, Delphi surveys tens of thousands of Facebook users, asking them a broad set of COVID-related questions, including whether they, or anyone in their household, are currently experiencing COVID-related symptoms.",smoothed_cli,CC BY,https://cmu.box.com/s/qfxplcdrcn9retfzx4zniyugbd9h3bos,"[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/fb-survey.html) +DB Source,Source Subdivision,Name,External Name,Description,Reference Signal,License,DUA,Link +chng,chng,Change Healthcare,Change Healthcare,Change Healthcare is a healthcare technology company that aggregates medical claims data from many healthcare providers. This source includes aggregated counts of claims with confirmed COVID-19 or COVID-related symptoms. All claims data has been de-identified in accordance with HIPAA privacy regulations. ,smoothed_outpatient_cli,CC BY-NC,https://cmu.box.com/s/cto4to822zecr3oyq1kkk9xmzhtq9tl2,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/chng.html) +covid-act-now,covid-act-now,Covid Act Now (CAN),Covid Act Now (CAN),"COVID Act Now (CAN) tracks COVID-19 testing statistics, such as positivity rates and total tests performed. This source only includes CAN data from the CDC's COVID-19 Integrated County View.",pcr_specimen_total_tests,CC BY-NC,,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/covid-act-now.html) +doctor-visits,doctor-visits,Doctor Visits From Claims,Optum,"Information about outpatient visits, provided to us by health system partners. Using outpatient claims counts, we estimate the percentage of COVID-related doctor's visits in a given location, on a given day.",smoothed_cli,CC BY,https://cmu.box.com/s/l2tz6kmiws6jyty2azwb43poiepz0565,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/doctor-visits.html) +fb-survey,fb-survey,Delphi US COVID-19 Trends and Impact Survey,Delphi/Meta,"We conduct the Delphi US COVID-19 Trends and Impact Survey (CTIS) in partnership with Facebook. Every day, Delphi surveys tens of thousands of Facebook users, asking them a broad set of COVID-related questions, including whether they, or anyone in their household, are currently experiencing COVID-related symptoms.",smoothed_cli,CC BY,https://cmu.box.com/s/qfxplcdrcn9retfzx4zniyugbd9h3bos,"[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/fb-survey.html) [Survey documentation](https://cmu-delphi.github.io/delphi-epidata/symptom-survey/) [Question text](https://cmu-delphi.github.io/delphi-epidata/symptom-survey/coding.html)" -google-symptoms,google-symptoms,Google Symptoms Search Trends,"Google's [COVID-19 Search Trends symptoms dataset](http://goo.gle/covid19symptomdataset) reflects the relative volume of Google searches for a broad set of symptoms, signs and health conditions. This source includes signals for 7 symptom sets: 6 sets of COVID-related symptoms, and 1 set of control symptoms unrelated to COVID-19. +google-symptoms,google-symptoms,Google Symptoms Search Trends,Google,"Google's [COVID-19 Search Trends symptoms dataset](http://goo.gle/covid19symptomdataset) reflects the relative volume of Google searches for a broad set of symptoms, signs and health conditions. This source includes signals for 7 symptom sets: 6 sets of COVID-related symptoms, and 1 set of control symptoms unrelated to COVID-19. Because of the way this dataset is constructed, values are comparable across signals in the same location, but not across geographic regions, even within the same signal. Use caution in any geographic analyses.",s05_smoothed_search,"To download or use the data, you must agree to the Google [Terms of Service](https://policies.google.com/terms)",,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/google-symptoms.html) -hhs,hhs,U.S. Department of Health & Human Services,The US Department of Health & Human Services (HHS) publishes several datasets on patient impact and hospital capacity. This source includes only adult and pediatric hospital admissions with confirmed and suspected COVID-19 or confirmed influenza.,confirmed_admissions_1d,[Public Domain US Government](https://www.usa.gov/government-works),,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/hhs.html) -hospital-admissions,hospital-admissions,Hospital Admissions From Claims,"Information about hospital admissions, provided to us by health system partners. Using inpatient claim counts, we estimate the percentage of new hospital admissions with a COVID-associated diagnosis code in a given location, on a given day. +hhs,hhs,U.S. Department of Health & Human Services,U.S. Department of Health & Human Services,The US Department of Health & Human Services (HHS) publishes several datasets on patient impact and hospital capacity. This source includes only adult and pediatric hospital admissions with confirmed and suspected COVID-19 or confirmed influenza.,confirmed_admissions_1d,[Public Domain US Government](https://www.usa.gov/government-works),,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/hhs.html) +hospital-admissions,hospital-admissions,Hospital Admissions From Claims,Optum,"Information about hospital admissions, provided to us by health system partners. Using inpatient claim counts, we estimate the percentage of new hospital admissions with a COVID-associated diagnosis code in a given location, on a given day. See also our [Health & Human Services](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/hhs.html) data source for official COVID hospitalization reporting from the US Department of Health & Human Services.",smoothed_covid19_from_claims,CC BY,https://cmu.box.com/s/l2tz6kmiws6jyty2azwb43poiepz0565,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/hospital-admissions.html) -indicator-combination,indicator-combination-cases-deaths,Composite COVID Cases & Deaths,"This source combines cases & deaths data from JHU and USA Facts, using JHU for Puerto Rico, and USA Facts everywhere else. It is not a primary source.",confirmed_incidence_num,CC BY,,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/indicator-combination.html#compositional-signals-confirmed-cases-and-deaths) -jhu-csse,jhu-csse,Johns Hopkins University,Johns Hopkins University's Center for Systems Science and Engineering (JHU-CSSE) reports confirmed COVID-19 cases and deaths.,confirmed_incidence_num,CC BY,,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/jhu-csse.html) -quidel,quidel-covid-ag,Quidel Inc. (COVID),"Quidel, Inc. manufactures diagnostic equipment for healthcare applications, and provides Delphi with anonymized data on tests and test results. This source includes antigen tests for COVID-19.",covid_ag_smoothed_pct_positive,CC BY,https://cmu.box.com/s/sax48yxnahllrnbqlq6wqxblg6lsyq24,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/quidel.html#covid-19-tests) -safegraph,safegraph-weekly,SafeGraph (Weekly),[SafeGraph](https://docs.safegraph.com/docs/weekly-patterns) compiles weekly insights on Points of Interest (POI) using anonymized location data from mobile phones. This source includes rates of restaurant and bar visits.,restaurants_visit_num,CC BY,https://cmu.box.com/s/m0p1wpet4vuvey7od83n70h0e97ky2kg,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/safegraph.html) -usa-facts,usa-facts,USAFacts,USAFacts is a nonprofit that aggregates and standardizes a vast array of data from US government agencies. This source reports confirmed COVID-19 cases and deaths.,confirmed_incidence_num,CC BY,,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/usa-facts.html) -ght,ght,Google Health Trends,"Google Health Trends tracks Google searches on health-related topics. This source includes Google Health Trends API results for overall searcher interest in a set of COVID-19 related terms about anosmia (lack of smell or taste), which emerged as a symptom of the coronavirus. No longer updated after March 8, 2021.",smoothed_search,unknown,,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/ght.html) -google-survey,google-survey,Google Symptom Surveys,"Delphi ran symptom surveys using a Google tool which collects responses through publisher websites, Google's Opinions Reward app, and similar applications. No longer updated after May 15, 2020.",smoothed_cli,CC BY,,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/google-survey.html) -indicator-combination,indicator-combination-nmf,Statistical Combination (NMF),"This source provides signals which are statistical combinations of other sources, calculated by Delphi. It is not a primary data source. No longer updated after March 17, 2021.",nmf_day_doc_fbs_ght,CC BY,,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/indicator-combination.html) -quidel,quidel-flu,Quidel Inc. (Flu),"Quidel, Inc. manufactures diagnostic equipment for healthcare applications, and provides Delphi with anonymized data on tests and test results. This source includes flu tests. No longer updated after May 19, 2020.",smoothed_pct_negative,CC BY,https://cmu.box.com/s/sax48yxnahllrnbqlq6wqxblg6lsyq24,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/quidel.html#flu-tests) -safegraph,safegraph-daily,SafeGraph (Daily),"[SafeGraph](https://docs.safegraph.com/docs/social-distancing-metrics) compiles daily mobility information using anonymized location data from mobile phones. This source includes a range of isolation/lockdown behaviors and home dwell time. No longer updated after April 19, 2021.",completely_home_prop,CC BY,https://cmu.box.com/s/m0p1wpet4vuvey7od83n70h0e97ky2kg,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/safegraph.html) -nchs-mortality,nchs-mortality,NCHS Mortality Data,"This data source of national provisional death counts is based on death certificate data received and coded by the National Center for Health Statistics ([NCHS](https://www.cdc.gov/nchs/nvss/vsrr/COVID19/index.htm)). This data is different from the death data available from USAFacts and JHU CSSE: deaths are reported by the date they occur, not the date they are reported by local health departments, and data is frequently reissued as additional death certificates from recent weeks are received and tabulated.",deaths_allcause_incidence_num,[NCHS Data Use Agreement](https://www.cdc.gov/nchs/data_access/restrictions.htm),,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/nchs-mortality.html) -dsew-cpr,dsew-cpr,COVID-19 Community Profile Report,"This data source is based on the daily report published by the Data Strategy and Execution Workgroup under the White House COVID-19 Team, which contains detailed daily-resolution figures on cases, deaths, testing, hospital admissions, healthcare resources, and vaccinations.",confirmed_admissions_covid_1d_7dav,[Public Domain US Government](https://www.usa.gov/government-works),,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/dsew-cpr.html) \ No newline at end of file +indicator-combination,indicator-combination-cases-deaths,Composite COVID Cases & Deaths,Delphi,"This source combines cases & deaths data from JHU and USA Facts, using JHU for Puerto Rico, and USA Facts everywhere else. It is not a primary source.",confirmed_incidence_num,CC BY,,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/indicator-combination.html#compositional-signals-confirmed-cases-and-deaths) +jhu-csse,jhu-csse,Johns Hopkins University,Johns Hopkins University,Johns Hopkins University's Center for Systems Science and Engineering (JHU-CSSE) reports confirmed COVID-19 cases and deaths.,confirmed_incidence_num,CC BY,,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/jhu-csse.html) +quidel,quidel-covid-ag,Quidel Inc. (COVID),Quidel,"Quidel, Inc. manufactures diagnostic equipment for healthcare applications, and provides Delphi with anonymized data on tests and test results. This source includes antigen tests for COVID-19.",covid_ag_smoothed_pct_positive,CC BY,https://cmu.box.com/s/sax48yxnahllrnbqlq6wqxblg6lsyq24,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/quidel.html#covid-19-tests) +safegraph,safegraph-weekly,SafeGraph (Weekly),SafeGraph,[SafeGraph](https://docs.safegraph.com/docs/weekly-patterns) compiles weekly insights on Points of Interest (POI) using anonymized location data from mobile phones. This source includes rates of restaurant and bar visits.,restaurants_visit_num,CC BY,https://cmu.box.com/s/m0p1wpet4vuvey7od83n70h0e97ky2kg,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/safegraph.html) +usa-facts,usa-facts,USAFacts,USAFacts,USAFacts is a nonprofit that aggregates and standardizes a vast array of data from US government agencies. This source reports confirmed COVID-19 cases and deaths.,confirmed_incidence_num,CC BY,,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/usa-facts.html) +ght,ght,Google Health Trends,Google,"Google Health Trends tracks Google searches on health-related topics. This source includes Google Health Trends API results for overall searcher interest in a set of COVID-19 related terms about anosmia (lack of smell or taste), which emerged as a symptom of the coronavirus. No longer updated after March 8, 2021.",smoothed_search,unknown,,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/ght.html) +google-survey,google-survey,Google Symptom Surveys,Google,"Delphi ran symptom surveys using a Google tool which collects responses through publisher websites, Google's Opinions Reward app, and similar applications. No longer updated after May 15, 2020.",smoothed_cli,CC BY,,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/google-survey.html) +indicator-combination,indicator-combination-nmf,Statistical Combination (NMF),Delphi,"This source provides signals which are statistical combinations of other sources, calculated by Delphi. It is not a primary data source. No longer updated after March 17, 2021.",nmf_day_doc_fbs_ght,CC BY,,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/indicator-combination-inactive.html) +quidel,quidel-flu,Quidel Inc. (Flu),Quidel,"Quidel, Inc. manufactures diagnostic equipment for healthcare applications, and provides Delphi with anonymized data on tests and test results. This source includes flu tests. No longer updated after May 19, 2020.",smoothed_pct_negative,CC BY,https://cmu.box.com/s/sax48yxnahllrnbqlq6wqxblg6lsyq24,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/quidel.html#flu-tests) +safegraph,safegraph-daily,SafeGraph (Daily),SafeGraph,"[SafeGraph](https://docs.safegraph.com/docs/social-distancing-metrics) compiles daily mobility information using anonymized location data from mobile phones. This source includes a range of isolation/lockdown behaviors and home dwell time. No longer updated after April 19, 2021.",completely_home_prop,CC BY,https://cmu.box.com/s/m0p1wpet4vuvey7od83n70h0e97ky2kg,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/safegraph.html) +nchs-mortality,nchs-mortality,NCHS Mortality Data,National Center for Health Statistics,"This data source of national provisional death counts is based on death certificate data received and coded by the National Center for Health Statistics ([NCHS](https://www.cdc.gov/nchs/nvss/vsrr/COVID19/index.htm)). This data is different from the death data available from USAFacts and JHU CSSE: deaths are reported by the date they occur, not the date they are reported by local health departments, and data is frequently reissued as additional death certificates from recent weeks are received and tabulated.",deaths_allcause_incidence_num,[NCHS Data Use Agreement](https://www.cdc.gov/nchs/data_access/restrictions.htm),,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/nchs-mortality.html) +dsew-cpr,dsew-cpr,COVID-19 Community Profile Report,US Government,"This data source is based on the daily report published by the Data Strategy and Execution Workgroup under the White House COVID-19 Team, which contains detailed daily-resolution figures on cases, deaths, testing, hospital admissions, healthcare resources, and vaccinations.",confirmed_admissions_covid_1d_7dav,[Public Domain US Government](https://www.usa.gov/government-works),,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/dsew-cpr.html) +nssp,nssp,National Syndromic Surveillance Program,Centers for Disease Control and Prevention National Syndromic Surveillance Program,"The National Syndromic Surveillance Program (NSSP) is an effort to track epidemiologically relevant conditions. This dataset in particular tracks emergency department (ED) visits arising from a subset of influenza-like illnesses, specifically influenza, COVID-19, and respiratory syncytial virus (RSV).",pct_ed_visits_covid,[Public Domain US Government](https://www.usa.gov/government-works),,[API Documentation](https://cmu-delphi.github.io/delphi-epidata/api/covidcast-signals/nssp.html) \ No newline at end of file diff --git a/src/server/endpoints/covidcast_utils/model.py b/src/server/endpoints/covidcast_utils/model.py index 6a62df796..627e366fb 100644 --- a/src/server/endpoints/covidcast_utils/model.py +++ b/src/server/endpoints/covidcast_utils/model.py @@ -196,7 +196,8 @@ def _load_data_sources(): data_sources_df: pd.DataFrame = pd.read_csv(_base_dir / "db_sources.csv") data_sources_df = data_sources_df.replace({np.nan: None}) data_sources_df.columns = map(_clean_column, data_sources_df.columns) - data_sources: List[DataSource] = [DataSource(**d) for d in data_sources_df.to_dict(orient="records")] + datasource_fields = {f.name for f in fields(DataSource)} + data_sources: List[DataSource] = [DataSource(**{k: v for k, v in d.items() if k in datasource_fields}) for d in data_sources_df.to_dict(orient="records")] data_sources_df.set_index("source") return data_sources, data_sources_df diff --git a/src/server/main.py b/src/server/main.py index 2ec07e5a5..9d308c8ac 100644 --- a/src/server/main.py +++ b/src/server/main.py @@ -6,7 +6,7 @@ from flask import request, send_file, Response, send_from_directory, jsonify -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger from ._config import URL_PREFIX, VERSION from ._common import app, set_compatibility_mode diff --git a/src/server/utils/dates.py b/src/server/utils/dates.py index 4d6c242c9..010a6d27f 100644 --- a/src/server/utils/dates.py +++ b/src/server/utils/dates.py @@ -10,7 +10,7 @@ from epiweeks import Week, Year from typing_extensions import TypeAlias -from delphi.epidata.common.logger import get_structured_logger +from delphi_utils import get_structured_logger # Alias for a sequence of date ranges (int, int) or date integers IntRange: TypeAlias = Union[Tuple[int, int], int]