diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 904a3bf69..9b48a931a 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,4 +1,6 @@ # Format geomap.py d4b056e7a4c11982324e9224c9f9f6fd5d5ec65c # Format test_geomap.py -79072dcdec3faca9aaeeea65de83f7fa5c00d53f \ No newline at end of file +79072dcdec3faca9aaeeea65de83f7fa5c00d53f +# Sort setup.py dependencies +6912077acba97e835aff7d0cd3d64309a1a9241d \ No newline at end of file diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 0534cbce2..13af97d63 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -16,28 +16,40 @@ jobs: if: github.event.pull_request.draft == false strategy: matrix: - packages: - [ - _delphi_utils_python, - changehc, - claims_hosp, - doctor_visits, - google_symptoms, - hhs_hosp, - nchs_mortality, - nwss_wastewater, - quidel_covidtest, - sir_complainsalot, - ] + include: + - package: "_delphi_utils_python" + dir: "delphi_utils" + - package: "changehc" + dir: "delphi_changehc" + - package: "claims_hosp" + dir: "delphi_claims_hosp" + - package: "doctor_visits" + dir: "delphi_doctor_visits" + - package: "google_symptoms" + dir: "delphi_google_symptoms" + - package: "hhs_hosp" + dir: "delphi_hhs" + - package: "nchs_mortality" + dir: "delphi_nchs_mortality" + - package: "nwss_wastewater" + dir: "delphi_nwss" + - package: "quidel_covidtest" + dir: "delphi_quidel_covidtest" + - package: "sir_complainsalot" + dir: "delphi_sir_complainsalot" defaults: run: - working-directory: ${{ matrix.packages }} + working-directory: ${{ matrix.package }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Python 3.8 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.8 + cache: "pip" + cache-dependency-path: "setup.py" - name: Install testing dependencies run: | python -m pip install --upgrade pip @@ -51,3 +63,8 @@ jobs: - name: Test run: | make test + - uses: akaihola/darker@v2.1.1 + with: + options: "--check --diff --isort --color" + src: "${{ matrix.package }}/${{ matrix.dir }}" + version: "~=2.1.1" diff --git a/README.md b/README.md index 049b3ad49..3d4f8d161 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ In early April 2020, Delphi developed a uniform data schema for [a new Epidata endpoint focused on COVID-19](https://cmu-delphi.github.io/delphi-epidata/api/covidcast.html). Our intent was to provide signals that would track in real-time and in fine geographic granularity all facets of the COVID-19 pandemic, aiding both nowcasting and forecasting. Delphi's long history in tracking and forecasting influenza made us uniquely situated to provide access to data streams not available anywhere else, including medical claims data, electronic medical records, lab test records, massive public surveys, and internet search trends. We also process commonly-used publicly-available data sources, both for user convenience and to provide data versioning for sources that do not track revisions themselves. -Each data stream arrives in a different format using a different delivery technique, be it sftp, an access-controlled API, or an email attachment. The purpose of each pipeline in this repository is to fetch the raw source data, extract informative aggregate signals, and output those signals---which we call **COVID-19 indicators**---in a common format for upload to the [COVIDcast API](https://cmu-delphi.github.io/delphi-epidata/api/covidcast.html). +Each data stream arrives in a different format using a different delivery technique, be it sftp, an access-controlled API, or an email attachment. The purpose of each pipeline in this repository is to fetch the raw source data, extract informative aggregate signals, and output those signals---which we call **COVID-19 indicators**---in a common format for upload to the [COVIDcast API](https://cmu-delphi.github.io/delphi-epidata/api/covidcast.html). For client access to the API, along with a variety of other utilities, see our [R](https://cmu-delphi.github.io/covidcast/covidcastR/) and [Python](https://cmu-delphi.github.io/covidcast/covidcast-py/html/) packages. @@ -13,18 +13,19 @@ For interactive visualizations (of a subset of the available indicators), see ou ## Organization Utilities: -* `_delphi_utils_python` - common behaviors -* `_template_python` & `_template_r` - starting points for new data sources -* `ansible` & `jenkins` - automated testing and deployment -* `sir_complainsalot` - a Slack bot to check for missing data + +- `_delphi_utils_python` - common behaviors +- `_template_python` & `_template_r` - starting points for new data sources +- `ansible` & `jenkins` - automated testing and deployment +- `sir_complainsalot` - a Slack bot to check for missing data Indicator pipelines: all remaining directories. -Each indicator pipeline includes its own documentation. +Each indicator pipeline includes its own documentation. -* Consult README.md for directions to install, lint, test, and run the pipeline for that indicator. -* Consult REVIEW.md for the checklist to use for code reviews. -* Consult DETAILS.md (if present) for implementation details, including handling of corner cases. +- Consult README.md for directions to install, lint, test, and run the pipeline for that indicator. +- Consult REVIEW.md for the checklist to use for code reviews. +- Consult DETAILS.md (if present) for implementation details, including handling of corner cases. ## Development @@ -35,6 +36,28 @@ Each indicator pipeline includes its own documentation. 3. Add new commits to your branch in response to feedback. 4. When approved, tag an admin to merge the PR. Let them know if this change should be released immediately, at a set future date, or if it can just go along for the ride whenever the next release happens. +### Linting and Formatting + +Each indicator has a `make lint` command to check for linting errors and a `make +format` command to incrementally format your code (using +[darker](https://github.com/akaihola/darker)). These are both automated with a +[Github Action](.github/workflows/python-ci.yml). + +If you get the error `ERROR:darker.git:fatal: Not a valid commit name `, +then it's likely because your local main branch is not up to date; either you +need to rebase or merge. Note that `darker` reads from `pyproject.toml` for +default settings. + +If the lines you change are in a file that uses 2 space indentation, `darker` +will indent the lines around your changes and not the rest, which will likely +break the code; in that case, you should probably just pass the whole file +through black. You can do that with the following command (using the same +virtual environment as above): + +```sh +env/bin/black +``` + ## Release Process The release process consists of multiple steps which can all be done via the GitHub website: diff --git a/_delphi_utils_python/.pylintrc b/_delphi_utils_python/.pylintrc deleted file mode 100644 index ad0180ed7..000000000 --- a/_delphi_utils_python/.pylintrc +++ /dev/null @@ -1,22 +0,0 @@ - -[MESSAGES CONTROL] - -disable=logging-format-interpolation, - too-many-locals, - too-many-arguments, - # Allow pytest functions to be part of a class. - no-self-use, - # Allow pytest classes to have one test. - too-few-public-methods - -[BASIC] - -# Allow arbitrarily short-named variables. -variable-rgx=([a-z_][a-z0-9_]*|[a-zA-Z]) -argument-rgx=([a-z_][a-z0-9_]*|[a-zA-Z]) -attr-rgx=([a-z_][a-z0-9_]*|[a-zA-Z]) - -[DESIGN] - -# Don't complain about pytest "unused" arguments. -ignored-argument-names=(_.*|run_as_module) \ No newline at end of file diff --git a/_delphi_utils_python/Makefile b/_delphi_utils_python/Makefile index dd9c5f37f..79d7f7943 100644 --- a/_delphi_utils_python/Makefile +++ b/_delphi_utils_python/Makefile @@ -14,9 +14,12 @@ install-ci: venv pip install . lint: - . env/bin/activate; pylint delphi_utils + . env/bin/activate; pylint delphi_utils --rcfile=../pyproject.toml . env/bin/activate; pydocstyle delphi_utils +format: + . env/bin/activate; darker delphi_utils + test: . env/bin/activate ;\ (cd tests && ../env/bin/pytest --cov=delphi_utils --cov-report=term-missing) diff --git a/_delphi_utils_python/delphi_utils/geomap.py b/_delphi_utils_python/delphi_utils/geomap.py index a313c754c..360673a37 100644 --- a/_delphi_utils_python/delphi_utils/geomap.py +++ b/_delphi_utils_python/delphi_utils/geomap.py @@ -3,7 +3,6 @@ Authors: Dmitry Shemetov @dshemetov, James Sharpnack @jsharpna, Maria Jahja """ -# pylint: disable=too-many-lines from os.path import join from collections import defaultdict from typing import Iterator, List, Literal, Optional, Set, Union @@ -13,7 +12,7 @@ from pandas.api.types import is_string_dtype -class GeoMapper: # pylint: disable=too-many-public-methods +class GeoMapper: """Geo mapping tools commonly used in Delphi. The GeoMapper class provides utility functions for translating between different @@ -624,7 +623,7 @@ def get_geos_within( if contained_geocode_type == "state": if container_geocode_type == "nation" and container_geocode == "us": crosswalk = self._crosswalks["state"]["state"] - return set(crosswalk["state_id"]) # pylint: disable=unsubscriptable-object + return set(crosswalk["state_id"]) if container_geocode_type == "hhs": crosswalk_hhs = self._crosswalks["fips"]["hhs"] crosswalk_state = self._crosswalks["fips"]["state"] diff --git a/_delphi_utils_python/delphi_utils/logger.py b/_delphi_utils_python/delphi_utils/logger.py index d04ff7673..d70ae4c8e 100644 --- a/_delphi_utils_python/delphi_utils/logger.py +++ b/_delphi_utils_python/delphi_utils/logger.py @@ -1,9 +1,10 @@ -"""Structured logger utility for creating JSON logs.""" +"""Structured logger utility for creating JSON logs. -# the Delphi group uses two ~identical versions of this file. -# try to keep them in sync with edits, for sanity. -# https://github.com/cmu-delphi/covidcast-indicators/blob/main/_delphi_utils_python/delphi_utils/logger.py # pylint: disable=line-too-long -# https://github.com/cmu-delphi/delphi-epidata/blob/dev/src/common/logger.py +The Delphi group uses two ~identical versions of this file. +Try to keep them in sync with edits, for sanity. + https://github.com/cmu-delphi/covidcast-indicators/blob/main/_delphi_utils_python/delphi_utils/logger.py + https://github.com/cmu-delphi/delphi-epidata/blob/dev/src/common/logger.py +""" import contextlib import logging diff --git a/_delphi_utils_python/delphi_utils/smooth.py b/_delphi_utils_python/delphi_utils/smooth.py index 503fcf1b2..d9c95b552 100644 --- a/_delphi_utils_python/delphi_utils/smooth.py +++ b/_delphi_utils_python/delphi_utils/smooth.py @@ -304,17 +304,11 @@ def left_gauss_linear_smoother(self, signal): n = len(signal) signal_smoothed = np.zeros_like(signal) # A is the regression design matrix - A = np.vstack([np.ones(n), np.arange(n)]).T # pylint: disable=invalid-name + A = np.vstack([np.ones(n), np.arange(n)]).T for idx in range(n): - weights = np.exp( - -((np.arange(idx + 1) - idx) ** 2) / self.gaussian_bandwidth - ) - AwA = np.dot( # pylint: disable=invalid-name - A[: (idx + 1), :].T * weights, A[: (idx + 1), :] - ) - Awy = np.dot( # pylint: disable=invalid-name - A[: (idx + 1), :].T * weights, signal[: (idx + 1)].reshape(-1, 1) - ) + weights = np.exp(-((np.arange(idx + 1) - idx) ** 2) / self.gaussian_bandwidth) + AwA = np.dot(A[: (idx + 1), :].T * weights, A[: (idx + 1), :]) + Awy = np.dot(A[: (idx + 1), :].T * weights, signal[: (idx + 1)].reshape(-1, 1)) try: beta = np.linalg.solve(AwA, Awy) signal_smoothed[idx] = np.dot(A[: (idx + 1), :], beta)[-1] @@ -389,9 +383,7 @@ def savgol_coeffs(self, nl, nr, poly_fit_degree): if nr > 0: warnings.warn("The filter is no longer causal.") - A = np.vstack( # pylint: disable=invalid-name - [np.arange(nl, nr + 1) ** j for j in range(poly_fit_degree + 1)] - ).T + A = np.vstack([np.arange(nl, nr + 1) ** j for j in range(poly_fit_degree + 1)]).T if self.gaussian_bandwidth is None: mat_inverse = np.linalg.inv(A.T @ A) @ A.T @@ -406,7 +398,7 @@ def savgol_coeffs(self, nl, nr, poly_fit_degree): coeffs[i] = (mat_inverse @ basis_vector)[0] return coeffs - def savgol_smoother(self, signal): # pylint: disable=inconsistent-return-statements + def savgol_smoother(self, signal): """Smooth signal with the savgol smoother. Returns a convolution of the 1D signal with the Savitzky-Golay coefficients, respecting diff --git a/_delphi_utils_python/delphi_utils/validator/dynamic.py b/_delphi_utils_python/delphi_utils/validator/dynamic.py index 6758086ab..9bc72ec1c 100644 --- a/_delphi_utils_python/delphi_utils/validator/dynamic.py +++ b/_delphi_utils_python/delphi_utils/validator/dynamic.py @@ -320,7 +320,6 @@ def create_dfs(self, geo_sig_df, api_df_or_error, checking_date, geo_type, signa # # These variables are interpolated into the call to `api_df_or_error.query()` # below but pylint doesn't recognize that. - # pylint: disable=unused-variable reference_start_date = recent_cutoff_date - self.params.max_check_lookbehind if signal_type in self.params.smoothed_signals: # Add an extra 7 days to the reference period. @@ -328,7 +327,6 @@ def create_dfs(self, geo_sig_df, api_df_or_error, checking_date, geo_type, signa timedelta(days=7) reference_end_date = recent_cutoff_date - timedelta(days=1) - # pylint: enable=unused-variable # Subset API data to relevant range of dates. reference_api_df = api_df_or_error.query( diff --git a/_delphi_utils_python/setup.py b/_delphi_utils_python/setup.py index 046dc5d3a..d4c9d2a0e 100644 --- a/_delphi_utils_python/setup.py +++ b/_delphi_utils_python/setup.py @@ -2,12 +2,13 @@ from setuptools import find_packages with open("README.md", "r") as f: - long_description = f.read() + long_description = f.read() required = [ "boto3", "covidcast", "cvxpy", + "darker[isort]~=2.1.1", "epiweeks", "freezegun", "gitpython", @@ -17,8 +18,8 @@ "pandas>=1.1.0", "pydocstyle", "pylint==2.8.3", - "pytest", "pytest-cov", + "pytest", "requests-mock", "slackclient", "structlog", diff --git a/_template_python/.pylintrc b/_template_python/.pylintrc deleted file mode 100644 index f30837c7e..000000000 --- a/_template_python/.pylintrc +++ /dev/null @@ -1,22 +0,0 @@ - -[MESSAGES CONTROL] - -disable=logging-format-interpolation, - too-many-locals, - too-many-arguments, - # Allow pytest functions to be part of a class. - no-self-use, - # Allow pytest classes to have one test. - too-few-public-methods - -[BASIC] - -# Allow arbitrarily short-named variables. -variable-rgx=[a-z_][a-z0-9_]* -argument-rgx=[a-z_][a-z0-9_]* -attr-rgx=[a-z_][a-z0-9_]* - -[DESIGN] - -# Don't complain about pytest "unused" arguments. -ignored-argument-names=(_.*|run_as_module) \ No newline at end of file diff --git a/_template_python/Makefile b/_template_python/Makefile index bc88f1fec..390113eef 100644 --- a/_template_python/Makefile +++ b/_template_python/Makefile @@ -17,9 +17,12 @@ install-ci: venv pip install . lint: - . env/bin/activate; pylint $(dir) + . env/bin/activate; pylint $(dir) --rcfile=../pyproject.toml . env/bin/activate; pydocstyle $(dir) +format: + . env/bin/activate; darker $(dir) + test: . env/bin/activate ;\ (cd tests && ../env/bin/pytest --cov=$(dir) --cov-report=term-missing) diff --git a/_template_python/setup.py b/_template_python/setup.py index ba1325b3c..d7bc44078 100644 --- a/_template_python/setup.py +++ b/_template_python/setup.py @@ -2,14 +2,15 @@ from setuptools import find_packages required = [ + "covidcast", + "darker[isort]~=2.1.1", + "delphi-utils", "numpy", "pandas", "pydocstyle", - "pytest", - "pytest-cov", "pylint==2.8.3", - "delphi-utils", - "covidcast" + "pytest-cov", + "pytest", ] setup( diff --git a/changehc/.pylintrc b/changehc/.pylintrc deleted file mode 100644 index c71c52434..000000000 --- a/changehc/.pylintrc +++ /dev/null @@ -1,24 +0,0 @@ - -[MESSAGES CONTROL] - -disable=logging-format-interpolation, - too-many-locals, - too-many-arguments, - # Allow pytest functions to be part of a class. - no-self-use, - # Allow pytest classes to have one test. - too-few-public-methods, - # Ignore - R0903, C0301, R0914, C0103, W1203, E0611, R0902, R0913, W0105, W0611, W1401 - -[BASIC] - -# Allow arbitrarily short-named variables. -variable-rgx=[a-z_][a-z0-9_]* -argument-rgx=[a-z_][a-z0-9_]* -attr-rgx=[a-z_][a-z0-9_]* - -[DESIGN] - -# Don't complain about pytest "unused" arguments. -ignored-argument-names=(_.*|run_as_module) \ No newline at end of file diff --git a/changehc/Makefile b/changehc/Makefile index bc88f1fec..390113eef 100644 --- a/changehc/Makefile +++ b/changehc/Makefile @@ -17,9 +17,12 @@ install-ci: venv pip install . lint: - . env/bin/activate; pylint $(dir) + . env/bin/activate; pylint $(dir) --rcfile=../pyproject.toml . env/bin/activate; pydocstyle $(dir) +format: + . env/bin/activate; darker $(dir) + test: . env/bin/activate ;\ (cd tests && ../env/bin/pytest --cov=$(dir) --cov-report=term-missing) diff --git a/changehc/delphi_changehc/download_ftp_files.py b/changehc/delphi_changehc/download_ftp_files.py index f85ef9944..dad47bb3f 100644 --- a/changehc/delphi_changehc/download_ftp_files.py +++ b/changehc/delphi_changehc/download_ftp_files.py @@ -1,7 +1,6 @@ """Download files from the specified ftp server.""" # standard -import datetime import functools from os import path diff --git a/changehc/delphi_changehc/sensor.py b/changehc/delphi_changehc/sensor.py index d1422567b..0449f07df 100644 --- a/changehc/delphi_changehc/sensor.py +++ b/changehc/delphi_changehc/sensor.py @@ -6,9 +6,6 @@ """ -# standard packages -import logging - # third party import numpy as np import pandas as pd diff --git a/changehc/delphi_changehc/update_sensor.py b/changehc/delphi_changehc/update_sensor.py index cb5b42a4b..a91dec5ac 100644 --- a/changehc/delphi_changehc/update_sensor.py +++ b/changehc/delphi_changehc/update_sensor.py @@ -5,18 +5,16 @@ Created: 2020-10-14 """ # standard packages -import logging from multiprocessing import Pool, cpu_count # third party import numpy as np import pandas as pd -from delphi_utils import GeoMapper, add_prefix, create_export_csv, Weekday +from delphi_utils import GeoMapper, Weekday, add_prefix, create_export_csv # first party from .config import Config -from .constants import SMOOTHED, SMOOTHED_ADJ, SMOOTHED_CLI, SMOOTHED_ADJ_CLI,\ - SMOOTHED_FLU, SMOOTHED_ADJ_FLU, NA +from .constants import SMOOTHED, SMOOTHED_ADJ, SMOOTHED_ADJ_CLI, SMOOTHED_ADJ_FLU, SMOOTHED_CLI, SMOOTHED_FLU from .sensor import CHCSensor @@ -173,10 +171,11 @@ def geo_reindex(self, data): unique_geo_ids = pd.unique(data_frame[geo]) data_frame.set_index([geo, Config.DATE_COL],inplace=True) # for each location, fill in all missing dates with 0 values - multiindex = pd.MultiIndex.from_product((unique_geo_ids, self.fit_dates), - names=[geo, Config.DATE_COL]) - assert (len(multiindex) <= (len(gmpr.get_geo_values(gmpr.as_mapper_name(geo))) * len(self.fit_dates)) - ), f"more loc-date pairs than maximum number of geographies x number of dates, length of multiindex is {len(multiindex)}, geo level is {geo}" + multiindex = pd.MultiIndex.from_product((unique_geo_ids, self.fit_dates), names=[geo, Config.DATE_COL]) + assert len(multiindex) <= (len(gmpr.get_geo_values(gmpr.as_mapper_name(geo))) * len(self.fit_dates)), ( + "more loc-date pairs than maximum number of geographies x number of dates, " + f"length of multiindex is {len(multiindex)}, geo level is {geo}" + ) # fill dataframe with missing dates using 0 data_frame = data_frame.reindex(multiindex, fill_value=0) diff --git a/changehc/setup.py b/changehc/setup.py index d46649391..1faebb6a5 100644 --- a/changehc/setup.py +++ b/changehc/setup.py @@ -2,18 +2,19 @@ from setuptools import find_packages required = [ + "boto3", + "covidcast", + "darker[isort]~=2.1.1", + "delphi-utils", + "moto~=4.2.14", "numpy", "pandas", + "paramiko", "pyarrow", "pydocstyle", - "pytest", - "pytest-cov", "pylint==2.8.3", - "delphi-utils", - "covidcast", - "boto3", - "moto~=4.2.14", - "paramiko" + "pytest-cov", + "pytest", ] setup( diff --git a/claims_hosp/.pylintrc b/claims_hosp/.pylintrc deleted file mode 100644 index 7fc2f5c30..000000000 --- a/claims_hosp/.pylintrc +++ /dev/null @@ -1,23 +0,0 @@ - -[MESSAGES CONTROL] - -disable=logging-format-interpolation, - too-many-locals, - too-many-arguments, - # Allow pytest functions to be part of a class. - no-self-use, - # Allow pytest classes to have one test. - too-few-public-methods, - - -[BASIC] - -# Allow arbitrarily short-named variables. -variable-rgx=[a-z_][a-z0-9_]* -argument-rgx=[a-z_][a-z0-9_]* -attr-rgx=[a-z_][a-z0-9_]* - -[DESIGN] - -# Don't complain about pytest "unused" arguments. -ignored-argument-names=(_.*|run_as_module) \ No newline at end of file diff --git a/claims_hosp/Makefile b/claims_hosp/Makefile index bc88f1fec..390113eef 100644 --- a/claims_hosp/Makefile +++ b/claims_hosp/Makefile @@ -17,9 +17,12 @@ install-ci: venv pip install . lint: - . env/bin/activate; pylint $(dir) + . env/bin/activate; pylint $(dir) --rcfile=../pyproject.toml . env/bin/activate; pydocstyle $(dir) +format: + . env/bin/activate; darker $(dir) + test: . env/bin/activate ;\ (cd tests && ../env/bin/pytest --cov=$(dir) --cov-report=term-missing) diff --git a/claims_hosp/delphi_claims_hosp/smooth.py b/claims_hosp/delphi_claims_hosp/smooth.py index a66bcc25c..56b132fa2 100644 --- a/claims_hosp/delphi_claims_hosp/smooth.py +++ b/claims_hosp/delphi_claims_hosp/smooth.py @@ -27,13 +27,11 @@ def left_gauss_linear(arr, bandwidth=Config.SMOOTHER_BANDWIDTH): """ n_rows = len(arr) out_arr = np.zeros_like(arr) - X = np.vstack([np.ones(n_rows), np.arange(n_rows)]).T # pylint: disable=invalid-name + X = np.vstack([np.ones(n_rows), np.arange(n_rows)]).T for idx in range(n_rows): weights = np.exp(-((np.arange(idx + 1) - idx) ** 2) / bandwidth) - # pylint: disable=invalid-name XwX = np.dot(X[: (idx + 1), :].T * weights, X[: (idx + 1), :]) Xwy = np.dot(X[: (idx + 1), :].T * weights, arr[: (idx + 1)].reshape(-1, 1)) - # pylint: enable=invalid-name try: beta = np.linalg.solve(XwX, Xwy) out_arr[idx] = np.dot(X[: (idx + 1), :], beta)[-1] diff --git a/claims_hosp/setup.py b/claims_hosp/setup.py index bc50a6414..490b38b99 100644 --- a/claims_hosp/setup.py +++ b/claims_hosp/setup.py @@ -2,16 +2,17 @@ from setuptools import find_packages required = [ + "covidcast", + "darker[isort]~=2.1.1", + "delphi-utils", "numpy", "pandas", - "pyarrow", "paramiko", + "pyarrow", "pydocstyle", - "pytest", - "pytest-cov", "pylint==2.8.3", - "delphi-utils", - "covidcast" + "pytest-cov", + "pytest", ] setup( diff --git a/doctor_visits/.pylintrc b/doctor_visits/.pylintrc deleted file mode 100644 index a14b269cc..000000000 --- a/doctor_visits/.pylintrc +++ /dev/null @@ -1,8 +0,0 @@ -[DESIGN] - -min-public-methods=0 - - -[MESSAGES CONTROL] - -disable=R0801, C0200, C0330, E1101, E0611, E1136, C0114, C0116, C0103, R0913, R0914, R0915, W1401, W1202, W1203, W0702 diff --git a/doctor_visits/Makefile b/doctor_visits/Makefile index bc88f1fec..390113eef 100644 --- a/doctor_visits/Makefile +++ b/doctor_visits/Makefile @@ -17,9 +17,12 @@ install-ci: venv pip install . lint: - . env/bin/activate; pylint $(dir) + . env/bin/activate; pylint $(dir) --rcfile=../pyproject.toml . env/bin/activate; pydocstyle $(dir) +format: + . env/bin/activate; darker $(dir) + test: . env/bin/activate ;\ (cd tests && ../env/bin/pytest --cov=$(dir) --cov-report=term-missing) diff --git a/doctor_visits/delphi_doctor_visits/run.py b/doctor_visits/delphi_doctor_visits/run.py index 93c346ee7..fd09c56d6 100644 --- a/doctor_visits/delphi_doctor_visits/run.py +++ b/doctor_visits/delphi_doctor_visits/run.py @@ -20,7 +20,7 @@ from .get_latest_claims_name import get_latest_filename -def run_module(params): +def run_module(params): # pylint: disable=too-many-statements """ Run doctor visits indicator. diff --git a/doctor_visits/delphi_doctor_visits/smooth.py b/doctor_visits/delphi_doctor_visits/smooth.py index 72f691942..d24f1b85f 100644 --- a/doctor_visits/delphi_doctor_visits/smooth.py +++ b/doctor_visits/delphi_doctor_visits/smooth.py @@ -22,7 +22,7 @@ def moving_avg(x, y, k=7): """ n = len(y) sy = np.zeros((n - k + 1, 1)) - for i in range(len(sy)): + for i in range(len(sy)): # pylint: disable=consider-using-enumerate sy[i] = np.mean(y[i : (i + k)]) return x[(k - 1) :], sy @@ -39,7 +39,7 @@ def padded_moving_avg(y, k=7): """ n = len(y) sy = np.zeros((n - k + 1, 1)) - for i in range(len(sy)): + for i in range(len(sy)): # pylint: disable=consider-using-enumerate sy[i] = np.mean(y[i : (i + k)]) # pad first k obs with 0 diff --git a/doctor_visits/setup.py b/doctor_visits/setup.py index faba7c670..fc291160d 100644 --- a/doctor_visits/setup.py +++ b/doctor_visits/setup.py @@ -2,14 +2,15 @@ from setuptools import find_packages required = [ + "darker[isort]~=2.1.1", + "delphi-utils", "numpy", "pandas", "paramiko", - "scikit-learn", - "pytest", - "pytest-cov", "pylint==2.8.3", - "delphi-utils" + "pytest-cov", + "pytest", + "scikit-learn", ] setup( diff --git a/google_symptoms/.pylintrc b/google_symptoms/.pylintrc deleted file mode 100644 index f337ecf9c..000000000 --- a/google_symptoms/.pylintrc +++ /dev/null @@ -1,8 +0,0 @@ -[DESIGN] - -min-public-methods=1 - - -[MESSAGES CONTROL] - -disable=R0801, E1101, E0611, C0114, C0116, C0103, R0913, R0914, W0702, W0707 diff --git a/google_symptoms/Makefile b/google_symptoms/Makefile index f6a5b7e63..6884278cf 100644 --- a/google_symptoms/Makefile +++ b/google_symptoms/Makefile @@ -17,9 +17,12 @@ install-ci: venv pip install . lint: - . env/bin/activate; pylint $(dir) + . env/bin/activate; pylint $(dir) --rcfile=../pyproject.toml . env/bin/activate; pydocstyle $(dir) +format: + . env/bin/activate; darker $(dir) + test: . env/bin/activate ; (cd tests && ../env/bin/pytest --cov=$(dir) --cov-report=term-missing) diff --git a/google_symptoms/delphi_google_symptoms/pull.py b/google_symptoms/delphi_google_symptoms/pull.py index 29def6b4e..d5921a3e4 100644 --- a/google_symptoms/delphi_google_symptoms/pull.py +++ b/google_symptoms/delphi_google_symptoms/pull.py @@ -67,7 +67,7 @@ def preprocess(df, level): try: df = df[KEEP_COLUMNS] except KeyError: - raise ValueError( + raise ValueError( # pylint: disable=raise-missing-from "Some necessary columns are missing. The dataset " "schema may have changed. Please investigate." ) diff --git a/google_symptoms/setup.py b/google_symptoms/setup.py index 91af03e64..ccba3c47a 100644 --- a/google_symptoms/setup.py +++ b/google_symptoms/setup.py @@ -2,17 +2,18 @@ from setuptools import find_packages required = [ + "darker[isort]~=2.1.1", + "db-dtypes", + "delphi-utils", + "freezegun", "mock", "numpy", + "pandas-gbq", "pandas", "pydocstyle", - "pytest", - "pytest-cov", "pylint==2.8.3", - "delphi-utils", - "freezegun", - "pandas-gbq", - "db-dtypes" + "pytest-cov", + "pytest", ] setup( diff --git a/hhs_hosp/.pylintrc b/hhs_hosp/.pylintrc deleted file mode 100644 index 58c6edbba..000000000 --- a/hhs_hosp/.pylintrc +++ /dev/null @@ -1,22 +0,0 @@ - -[MESSAGES CONTROL] - -disable=logging-format-interpolation, - too-many-locals, - too-many-arguments, - # Allow pytest functions to be part of a class. - no-self-use, - # Allow pytest classes to have one test. - too-few-public-methods - -[BASIC] - -# Allow arbitrarily short-named variables. -variable-rgx=[a-z_][a-z0-9_]* -argument-rgx=[a-z_][a-z0-9_]* -attr-rgx=[a-z_][a-z0-9_]* - -[DESIGN] - -# Don't complain about pytest "unused" arguments. -ignored-argument-names=(_.*|run_as_module) diff --git a/hhs_hosp/Makefile b/hhs_hosp/Makefile index ea591dcb5..69529feb7 100644 --- a/hhs_hosp/Makefile +++ b/hhs_hosp/Makefile @@ -17,9 +17,12 @@ install-ci: venv pip install . lint: - . env/bin/activate; pylint $(dir) + . env/bin/activate; pylint $(dir) --rcfile=../pyproject.toml . env/bin/activate; pydocstyle $(dir) +format: + . env/bin/activate; darker $(dir) + test: . env/bin/activate ;\ (cd tests && ../env/bin/pytest --cov=$(dir) --cov-report=term-missing) diff --git a/hhs_hosp/setup.py b/hhs_hosp/setup.py index b19bcbb42..90a685ac8 100644 --- a/hhs_hosp/setup.py +++ b/hhs_hosp/setup.py @@ -2,16 +2,17 @@ from setuptools import find_packages required = [ + "covidcast", + "darker[isort]~=2.1.1", + "delphi-epidata", + "delphi-utils", "freezegun", "numpy", "pandas", "pydocstyle", - "pytest", - "pytest-cov", "pylint==2.8.3", - "delphi-utils", - "covidcast", - "delphi-epidata" + "pytest-cov", + "pytest", ] setup( diff --git a/nchs_mortality/.pylintrc b/nchs_mortality/.pylintrc deleted file mode 100644 index c72b4e124..000000000 --- a/nchs_mortality/.pylintrc +++ /dev/null @@ -1,24 +0,0 @@ - -[MESSAGES CONTROL] - -disable=logging-format-interpolation, - too-many-locals, - too-many-arguments, - too-many-branches, - too-many-statements, - # Allow pytest functions to be part of a class. - no-self-use, - # Allow pytest classes to have one test. - too-few-public-methods - -[BASIC] - -# Allow arbitrarily short-named variables. -variable-rgx=[a-z_][a-z0-9_]* -argument-rgx=[a-z_][a-z0-9_]* -attr-rgx=[a-z_][a-z0-9_]* - -[DESIGN] - -# Don't complain about pytest "unused" arguments. -ignored-argument-names=(_.*|run_as_module) \ No newline at end of file diff --git a/nchs_mortality/Makefile b/nchs_mortality/Makefile index bc88f1fec..390113eef 100644 --- a/nchs_mortality/Makefile +++ b/nchs_mortality/Makefile @@ -17,9 +17,12 @@ install-ci: venv pip install . lint: - . env/bin/activate; pylint $(dir) + . env/bin/activate; pylint $(dir) --rcfile=../pyproject.toml . env/bin/activate; pydocstyle $(dir) +format: + . env/bin/activate; darker $(dir) + test: . env/bin/activate ;\ (cd tests && ../env/bin/pytest --cov=$(dir) --cov-report=term-missing) diff --git a/nchs_mortality/setup.py b/nchs_mortality/setup.py index 76915936b..3fe354ba4 100644 --- a/nchs_mortality/setup.py +++ b/nchs_mortality/setup.py @@ -2,16 +2,17 @@ from setuptools import find_packages required = [ + "darker[isort]~=2.1.1", + "delphi-utils", + "epiweeks", + "freezegun", "numpy", "pandas", "pydocstyle", - "pytest", - "pytest-cov", "pylint==2.8.3", - "delphi-utils", + "pytest-cov", + "pytest", "sodapy", - "epiweeks", - "freezegun", ] setup( diff --git a/nwss_wastewater/.pylintrc b/nwss_wastewater/.pylintrc deleted file mode 100644 index f30837c7e..000000000 --- a/nwss_wastewater/.pylintrc +++ /dev/null @@ -1,22 +0,0 @@ - -[MESSAGES CONTROL] - -disable=logging-format-interpolation, - too-many-locals, - too-many-arguments, - # Allow pytest functions to be part of a class. - no-self-use, - # Allow pytest classes to have one test. - too-few-public-methods - -[BASIC] - -# Allow arbitrarily short-named variables. -variable-rgx=[a-z_][a-z0-9_]* -argument-rgx=[a-z_][a-z0-9_]* -attr-rgx=[a-z_][a-z0-9_]* - -[DESIGN] - -# Don't complain about pytest "unused" arguments. -ignored-argument-names=(_.*|run_as_module) \ No newline at end of file diff --git a/nwss_wastewater/Makefile b/nwss_wastewater/Makefile index bc88f1fec..390113eef 100644 --- a/nwss_wastewater/Makefile +++ b/nwss_wastewater/Makefile @@ -17,9 +17,12 @@ install-ci: venv pip install . lint: - . env/bin/activate; pylint $(dir) + . env/bin/activate; pylint $(dir) --rcfile=../pyproject.toml . env/bin/activate; pydocstyle $(dir) +format: + . env/bin/activate; darker $(dir) + test: . env/bin/activate ;\ (cd tests && ../env/bin/pytest --cov=$(dir) --cov-report=term-missing) diff --git a/nwss_wastewater/setup.py b/nwss_wastewater/setup.py index f2cce8cb3..26f1b7324 100644 --- a/nwss_wastewater/setup.py +++ b/nwss_wastewater/setup.py @@ -2,16 +2,17 @@ from setuptools import find_packages required = [ + "darker[isort]~=2.1.1", + "delphi-utils", + "epiweeks", + "freezegun", "numpy", "pandas", "pydocstyle", - "pytest", - "pytest-cov", "pylint==2.8.3", - "delphi-utils", + "pytest-cov", + "pytest", "sodapy", - "epiweeks", - "freezegun", ] setup( diff --git a/pyproject.toml b/pyproject.toml index 9a31b63a0..2ca230476 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,37 @@ [tool.black] line-length = 120 target-version = ['py38'] -include = '_delphi_utils_python' + +[tool.darker] +revision = 'origin/main...' +color = true +isort = true + +[tool.isort] +profile = "black" +known_third_party = ["pytest"] + +[tool.pylint] +[tool.pylint.main] +max-line-length = 120 +disable = [ + 'logging-format-interpolation', + # Allow pytest functions to be part of a class + 'no-self-use', + 'too-many-locals', + 'too-many-arguments', + 'too-many-branches', + 'too-many-statements', + # Allow pytest classes to have one test + 'too-few-public-methods', +] +enable = 'useless-suppression' + +[tool.pylint.basic] +# Allow arbitrarily short-named variables. +variable-rgx = '[A-Za-z_][a-z0-9_]*' +argument-rgx = '[A-Za-z_][a-z0-9_]*' +attr-rgx = '[A-Za-z_][a-z0-9_]*' + +[tool.pylint.design] +ignored-argument-names = ['(_.*|run_as_module)'] diff --git a/quidel_covidtest/.pylintrc b/quidel_covidtest/.pylintrc deleted file mode 100644 index 29bd9aac2..000000000 --- a/quidel_covidtest/.pylintrc +++ /dev/null @@ -1,24 +0,0 @@ - -[MESSAGES CONTROL] - -disable=logging-format-interpolation, - too-many-locals, - too-many-arguments, - too-many-branches, - # Allow pytest functions to be part of a class. - no-self-use, - # Allow pytest classes to have one test. - too-few-public-methods -enable=useless-suppression - -[BASIC] - -# Allow arbitrarily short-named variables. -variable-rgx=[a-z_][a-z0-9_]* -argument-rgx=[a-z_][a-z0-9_]* -attr-rgx=[a-z_][a-z0-9_]* - -[DESIGN] - -# Don't complain about pytest "unused" arguments. -ignored-argument-names=(_.*|run_as_module) diff --git a/quidel_covidtest/Makefile b/quidel_covidtest/Makefile index bc88f1fec..390113eef 100644 --- a/quidel_covidtest/Makefile +++ b/quidel_covidtest/Makefile @@ -17,9 +17,12 @@ install-ci: venv pip install . lint: - . env/bin/activate; pylint $(dir) + . env/bin/activate; pylint $(dir) --rcfile=../pyproject.toml . env/bin/activate; pydocstyle $(dir) +format: + . env/bin/activate; darker $(dir) + test: . env/bin/activate ;\ (cd tests && ../env/bin/pytest --cov=$(dir) --cov-report=term-missing) diff --git a/quidel_covidtest/setup.py b/quidel_covidtest/setup.py index 369ac30c0..c2791930f 100644 --- a/quidel_covidtest/setup.py +++ b/quidel_covidtest/setup.py @@ -2,18 +2,19 @@ from setuptools import find_packages required = [ + "covidcast", + "darker[isort]~=2.1.1", + "delphi-utils", + "imap-tools", "numpy", + "openpyxl", "pandas", "pyarrow", "pydocstyle", - "pytest", - "pytest-cov", "pylint==2.8.3", - "delphi-utils", - "imap-tools", + "pytest-cov", + "pytest", "xlrd==1.2.0", - "covidcast", - "openpyxl" ] setup( diff --git a/sir_complainsalot/Makefile b/sir_complainsalot/Makefile index bc88f1fec..390113eef 100644 --- a/sir_complainsalot/Makefile +++ b/sir_complainsalot/Makefile @@ -17,9 +17,12 @@ install-ci: venv pip install . lint: - . env/bin/activate; pylint $(dir) + . env/bin/activate; pylint $(dir) --rcfile=../pyproject.toml . env/bin/activate; pydocstyle $(dir) +format: + . env/bin/activate; darker $(dir) + test: . env/bin/activate ;\ (cd tests && ../env/bin/pytest --cov=$(dir) --cov-report=term-missing) diff --git a/sir_complainsalot/delphi_sir_complainsalot/run.py b/sir_complainsalot/delphi_sir_complainsalot/run.py index 962fa1bc3..a1555b9c2 100644 --- a/sir_complainsalot/delphi_sir_complainsalot/run.py +++ b/sir_complainsalot/delphi_sir_complainsalot/run.py @@ -48,7 +48,7 @@ def run_module(): elapsed_time_in_seconds = elapsed_time_in_seconds) -def split_complaints(complaints, n=49): # pylint: disable=invalid-name +def split_complaints(complaints, n=49): """Yield successive n-sized chunks from complaints list.""" for i in range(0, len(complaints), n): yield complaints[i:i + n] diff --git a/sir_complainsalot/setup.py b/sir_complainsalot/setup.py index c51253104..157c001b2 100644 --- a/sir_complainsalot/setup.py +++ b/sir_complainsalot/setup.py @@ -2,13 +2,14 @@ from setuptools import find_packages required = [ + "covidcast", + "darker[isort]~=2.1.1", + "delphi-utils", "pandas", - "pytest", - "pytest-cov", "pylint==2.8.3", - "delphi-utils", + "pytest-cov", + "pytest", "slackclient", - "covidcast" ] setup(