From f5408ccaa6b279d4619e1b34b3ad74f2759ba0e4 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Wed, 25 Oct 2023 10:49:46 -0700 Subject: [PATCH] feat: add incremental linting --- .github/workflows/ci.yaml | 2 +- .github/workflows/lint.yaml | 40 ++++++++++++++++++++++++++++++++ .gitignore | 2 ++ README.md | 14 +++++++++++ pyproject.toml | 11 ++++++--- requirements.lint.txt | 6 +++++ tasks.py | 46 ++++++++++++++++++++++++++++++++++--- 7 files changed, 114 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/lint.yaml create mode 100644 requirements.lint.txt diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fd2d5fb4b..b46787e49 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -65,7 +65,7 @@ jobs: docker network create --driver bridge delphi-net docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata docker run --rm -d -p 6379:6379 --network delphi-net --env "REDIS_PASSWORD=1234" --name delphi_redis delphi_redis - + - run: | wget https://raw.githubusercontent.com/eficode/wait-for/master/wait-for diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 000000000..504739c39 --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,40 @@ +name: Lint + +on: + pull_request: + branches: + - dev + +jobs: + darker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-python@v4 + - uses: akaihola/darker@1.7.2 + with: + options: "--check --diff --isort --color" + version: "~=1.7.2" + + lint-diffs: + runs-on: ubuntu-latest + steps: + - name: Commit Range + id: commit-range + uses: akaihola/darker/.github/actions/commit-range@1.7.1 + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-python@v4 + with: + python-version: '3.8' + cache: 'pip' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.lint.txt + - name: Lint + run: | + inv lint --diff --no-format --revision=${{ steps.commit-range.outputs.commit-range }} diff --git a/.gitignore b/.gitignore index 7daea21f0..9eb8b80ca 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ __pycache__/ /node_modules .mypy_cache /missing_db_signals.csv +venv +env diff --git a/README.md b/README.md index 170c9a4ef..4e6990df7 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,20 @@ $ cd repos $ pip install -e . --config-settings editable_mode=strict ``` +## Linting and Formatting + +We use [darker](https://github.com/akaihola/darker) and +[lint-diffs](https://github.com/AtakamaLLC/lint-diffs/) to incrementally bring +this repo up to PEP8 compliance. There is CI to ensure that all new code is +compliant. To run the linter locally: + +```sh +# Install lint dependencies +pip install -r requirements.lint.txt +# Run lint +inv lint +``` + # COVIDcast At the present, our primary focus is developing and expanding the diff --git a/pyproject.toml b/pyproject.toml index a4399ca9b..2493f8cfa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,16 @@ [tool.black] -line-length = 100 +line-length = 120 target-version = ['py38'] -include = 'server,tests/server' + +[tool.isort] +profile = "black" +known_third_party = ["pytest"] [tool.pylint] + # Settings copied from + # cmu-delphi/covidcast-indicators/_delphi_utils_python/.pylintrc [tool.pylint.'MESSAGES CONTROL'] - max-line-length = 100 + max-line-length = 120 disable = [ 'logging-format-interpolation', # Allow pytest functions to be part of a class diff --git a/requirements.lint.txt b/requirements.lint.txt new file mode 100644 index 000000000..b00aa20f4 --- /dev/null +++ b/requirements.lint.txt @@ -0,0 +1,6 @@ +darker[flynt]~=1.7.2 +invoke>=1.4.1 +isort==5.12.0 +lint_diffs==0.1.22 +pylint==2.8.3 +requests \ No newline at end of file diff --git a/tasks.py b/tasks.py index fd7115f7e..7542317c5 100644 --- a/tasks.py +++ b/tasks.py @@ -1,3 +1,8 @@ +"""Repo tasks.""" +import pathlib +import sys + +import requests from invoke import task @@ -12,9 +17,6 @@ def update_gdoc( sources_url="https://docs.google.com/spreadsheets/d/e/2PACX-1vRfXo-qePhrYGAoZqewVnS1kt9tfnUTLgtkV7a-1q7yg4FoZk0NNGuB1H6k10ah1Xz5B8l1S1RB17N6/pub?gid=0&single=true&output=csv", signal_url="https://docs.google.com/spreadsheets/d/e/2PACX-1vRfXo-qePhrYGAoZqewVnS1kt9tfnUTLgtkV7a-1q7yg4FoZk0NNGuB1H6k10ah1Xz5B8l1S1RB17N6/pub?gid=329338228&single=true&output=csv", ): - import requests - import pathlib - base_dir = pathlib.Path("./src/server/endpoints/covidcast_utils/") def _migrate_file(url: str, filename: str): @@ -26,3 +28,41 @@ def _migrate_file(url: str, filename: str): _migrate_file(sources_url, "db_sources.csv") _migrate_file(signal_url, "db_signals.csv") + + +@task +def lint(c, incremental=True, format=True, revision="origin/dev", diff=False): # pylint: disable=redefined-builtin + """Lint and format. + + Additional linter settings can be found in `pyproject.toml` (this invocation + will use those settings as well). + + Parameters + ---------- + incremental : bool + Only lint changed files. + format : bool + Apply formatting changes. + revision : str + Revision to compare against. + diff : bool + Only show formatting changes, do not apply. + """ + if not incremental: + reponse = input("This will format all files in this repo, continue? [y/N]") + if reponse.lower() not in ("y", "yes"): + return + + diff = "--diff" if diff else "" + if incremental: + if format: + c.run(f"darker --revision {revision}... {diff} --flynt --isort --color --check .") + out = c.run(f"git diff -U0 {revision} | lint-diffs") + if out.stdout.strip() != "=== pylint: mine=0, always=0": + print(out.stdout) + sys.exit(1) + else: + if format: + c.run(f"black {diff} .") + c.run(f"isort {diff} .") + c.run("pylint src/ tests/ integrations/")