Python formatting and linting (G-Research#249)

* Add pipenv basic environment with mlflow, black, and isort
* Install black and isort VS Code extensions
* Setup VS Code for automatic Python formatting
* Add Makefile targets for formatting and linting
* Add Python linting to CI
* Format Python files

.devcontainer/devcontainer.json

@@ -22,7 +22,9 @@
 				"golang.Go",
 				"ms-python.python",
 				"pbkit.vscode-pbkit",
-				"redhat.vscode-yaml"
+				"redhat.vscode-yaml",
+				"ms-python.black-formatter",
+				"ms-python.isort"
 			]
 		}
 	},

.github/workflows/ci.yml

@@ -12,9 +12,9 @@ permissions:
   contents: read
 
 jobs:
-  lint:
+  go-lint:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
-    name: Lint
+    name: Go Lint
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -61,9 +61,28 @@ jobs:
           version: "2022.1.3"
           build-tags: ${{ steps.tags.outputs.tags }}
 
-  golang-unit-tests:
+  python-lint:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
-    name: Golang Unit Tests
+    name: Python Lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install pipenv
+        run: pip install pipenv==2022.12.19
+
+      - name: Run Python linters
+        run: make python-lint
+
+  go-unit-tests:
+    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
+    name: Go Unit Tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -80,12 +99,12 @@ jobs:
       - name: Generate mocks
         run: make mocks-generate
 
-      - name: Run Golang Unit Tests
+      - name: Run Go Unit Tests
         run: make test-go-unit
 
-  golang-integration-tests:
+  go-integration-tests:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
-    name: Golang Integration Tests
+    name: Go Integration Tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -174,6 +193,9 @@ jobs:
         with:
           python-version: "3.11"
 
+      - name: Install pipenv
+        run: pip install pipenv==2022.12.19
+
       - name: Install arm64 cross-compilation toolchain on Linux
         if: matrix.os == 'linux' && matrix.arch == 'arm64'
         run: |
@@ -266,9 +288,10 @@ jobs:
   all-required-checks-done:
     name: All required checks done
     needs:
-      - lint
-      - golang-unit-tests
-      - golang-integration-tests
+      - go-lint
+      - python-lint
+      - go-unit-tests
+      - go-integration-tests
       - python-integration-tests
       - build
       - build-image

.vscode/settings.json

@@ -8,5 +8,19 @@
     "go.lintTool": "golangci-lint",
     "go.lintFlags": [
         "--fast"
+    ],
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter",
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": true
+        }
+    },
+    "black-formatter.args": [
+        "--line-length",
+        "120"
+    ],
+    "isort.args": [
+        "--profile",
+        "black"
     ]
 }

Makefile

@@ -84,10 +84,27 @@ go-dist: go-build ## archive app binary.
 #
 # Python targets.
 #
+.PHONY: python-env
+python-env: ## create python virtual environment.
+	@echo '>>> Creating python virtual environment.'
+	@pipenv sync
+
 .PHONY: python-dist
-python-dist: go-build ## build python wheels.
+python-dist: go-build python-env ## build python wheels.
 	@echo '>>> Building Python Wheels.'
-	@VERSION=$(VERSION) python3 -m pip wheel ./python --wheel-dir=wheelhouse --no-deps
+	@VERSION=$(VERSION) pipenv run python3 -m pip wheel ./python --wheel-dir=wheelhouse --no-deps
+
+.PHONY: python-format
+python-format: python-env ## format python code.
+	@echo '>>> Formatting python code.'
+	@pipenv run black --line-length 120 .
+	@pipenv run isort --profile black .
+
+.PHONY: python-lint
+python-lint: python-env ## check python code formatting.
+	@echo '>>> Checking python code formatting.'
+	@pipenv run black --check --line-length 120 .
+	@pipenv run isort --check-only --profile black .
 
 #
 # Tests targets.
@@ -181,6 +198,9 @@ build: go-build ## build the go components
 PHONY: dist
 dist: go-dist python-dist ## build the software archives
 
+PHONY: format
+format: go-format python-format ## format the code
+
 PHONY: run
 run: build ## run the FastTrackML server
 	@echo ">>> Running the FasttrackML server."

Pipfile

@@ -0,0 +1,14 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+mlflow-skinny = "==2.6.0"
+black = "*"
+isort = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.11"