Initial commit

.gitattributes

@@ -0,0 +1,2 @@
+# Exclude jupyter notebooks for language stats.
+*.ipynb linguist-vendored

.github/pull_request_template.md

@@ -0,0 +1,7 @@
+# PR Type ([Feature | Fix | Documentation])
+
+## Short Description
+...
+
+## Tests Added
+...

.github/workflows/code_checks.yml

@@ -0,0 +1,47 @@
+name: Code checks
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - .pre-commit-config.yaml
+      - .github/workflows/code_checks.yml
+      - '**.py'
+      - poetry.lock
+      - pyproject.toml
+      - requirements.txt
+      - '**.ipynb'
+  pull_request:
+    branches:
+      - master
+    paths:
+      - .pre-commit-config.yaml
+      - .github/workflows/code_checks.yml
+      - '**.py'
+      - poetry.lock
+      - pyproject.toml
+      - requirements.txt
+      - '**.ipynb'
+
+jobs:
+  run-code-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install apt dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install libcurl4-openssl-dev libssl-dev
+      - uses: actions/checkout@v3
+      - name: Install poetry
+        run: pipx install poetry
+      - uses: actions/setup-python@v4.3.0
+        with:
+          python-version: 3.9.7
+          cache: 'poetry'
+      - name: Install dependencies and check code
+        run: |
+          poetry env use 3.9.7
+          source $(poetry env info --path)/bin/activate
+          poetry install
+          pre-commit run --all-files

.github/workflows/docs_build.yml

@@ -0,0 +1,39 @@
+name: Build documentation and coverage report
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - .pre-commit-config.yaml
+      - .github/workflows/docs_build.yml
+      - '**.py'
+      - poetry.lock
+      - pyproject.toml
+      - requirements.txt
+      - '**.rst'
+      - '**.md'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install apt dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install libcurl4-openssl-dev libssl-dev pandoc
+      - uses: actions/checkout@v3
+      - name: Install dependencies, build docs and coverage report
+        run: pipx install poetry
+      - uses: actions/setup-python@v4.3.0
+        with:
+          python-version: 3.9.7
+          cache: 'poetry'
+      - run: |
+          poetry env use 3.9.7
+          source $(poetry env info --path)/bin/activate
+          poetry install
+          pandoc README.md -f markdown -t rst -s -o docs/source/intro.rst
+          cd docs && make html
+          cd .. && coverage run -m pytest && coverage html && cp htmlcov/* docs/build/html/_static/
+          mv docs/build/html/_static/index.html docs/build/html/_static/coverage_report.html

.github/workflows/docs_deploy.yml

@@ -0,0 +1,45 @@
+name: Deploy documentation
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - .pre-commit-config.yaml
+      - .github/workflows/docs_build.yml
+      - .github/workflows/docs_deploy.yml
+      - '**.py'
+      - poetry.lock
+      - pyproject.toml
+      - requirements.txt
+      - '**.rst'
+      - '**.md'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install apt dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install libcurl4-openssl-dev libssl-dev pandoc
+      - uses: actions/checkout@v3
+      - name: Install dependencies, build docs and coverage report
+        run: pipx install poetry
+      - uses: actions/setup-python@v4.3.0
+        with:
+          python-version: 3.9.7
+          cache: 'poetry'
+      - run: |
+          poetry env use 3.9.7
+          source $(poetry env info --path)/bin/activate
+          poetry install
+          pandoc README.md -f markdown -t rst -s -o docs/source/intro.rst
+          cd docs && make html
+          cd .. && coverage run -m pytest && coverage html && cp htmlcov/* docs/build/html/_static/
+          mv docs/build/html/_static/index.html docs/build/html/_static/coverage_report.html
+      - name: Deploy to Github pages
+        uses: JamesIves/github-pages-deploy-action@4.1.1
+        with:
+          branch: github_pages
+          folder: docs/build/html

.gitignore

@@ -0,0 +1,24 @@
+*__pycache__*
+*.log
+docs/build
+.python-version
+*.DS_Store
+htmlcov
+.coverage
+venv
+.ipynb_checkpoints
+*.pt
+*.csv
+mlruns
+.profile
+.env
+*.html
+_extract
+*.gzip
+*checkpoint*
+*.parquet
+*.npy
+*.pkl
+*.npz
+*.csv
+*.pickle

.pre-commit-config.yaml

@@ -0,0 +1,91 @@
+exclude: |
+    (?x)(
+        ^drift_detection/|
+        ^notebooks/misc/batching_explore.ipynb
+    )
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0  # Use the ref you want to point at
+    hooks:
+    - id: trailing-whitespace
+    - id: check-ast
+    - id: check-builtin-literals
+    - id: check-docstring-first
+    - id: check-executables-have-shebangs
+    - id: debug-statements
+    - id: end-of-file-fixer
+    - id: mixed-line-ending
+      args: [--fix=lf]
+    - id: requirements-txt-fixer
+    - id: trailing-whitespace
+    - id: check-yaml
+
+  - repo: https://github.com/ambv/black
+    rev: 22.3.0
+    hooks:
+    - id: black
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.5.2
+    hooks:
+    - id: isort
+
+  - repo: https://github.com/myint/docformatter
+    rev: v1.3.1
+    hooks:
+    - id: docformatter
+      args: [--in-place, --wrap-summaries=88, --wrap-descriptions=88, --blank]
+      files: ".*.py$"
+
+  - repo: local
+    hooks:
+    - id: flake8
+      name: flake8
+      language: python
+      entry: pflake8
+      files: ".*.py$"
+
+  - repo: https://github.com/PyCQA/pydocstyle
+    rev: 6.1.1
+    hooks:
+    - id: pydocstyle
+      args: [--convention=numpy]
+      additional_dependencies: [toml]
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.942
+    hooks:
+    - id: mypy
+      args: [--namespace-packages, --explicit-package-bases]
+      additional_dependencies: [types-python-dateutil==2.8.8, types-PyYAML==6.0.5]
+
+  - repo: local
+    hooks:
+    - id: pylint
+      name: pylint
+      language: python
+      entry: pylint
+      files: ".*.py$"
+
+  - repo: local
+    hooks:
+    - id: pytest
+      name: pytest
+      entry: pytest
+      language: system
+      pass_filenames: false
+      always_run: true
+
+  - repo: https://github.com/nbQA-dev/nbQA
+    rev: 1.3.1
+    hooks:
+    - id: nbqa-black
+    - id: nbqa-isort
+    - id: nbqa-check-ast
+    - id: nbqa-flake8
+      # - id: nbqa-mypy
+
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.6.0
+    hooks:
+    - id: nbstripout

CHANGELOG.md

@@ -0,0 +1,34 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+Currently this project is early in development, so the semantic versioning is more to just keep track of development changes, not releases.
+
+## [0.0.1] - 2022-01-13 (6e5b1a00cddfe4cdaf58b41d82ad948a204dbc1d)
+This is first development version after Maria's handover of codebase.
+
+### Added
+- Code formatting using black
+- Cleanup README
+- Installation of dependencies using both pip and Conda
+
+### Fixed
+- Jupyter notebooks sourcing environment variables using python-dotenv package
+- postgres username automatically sourced using $USER env variable
+
+
+## [0.0.2] - 2022-02-01 (d397ae701a2b67ad023f0d4fd4c9d6d015e55d29)
+
+### Added
+- Add pre-commit hooks for static code analysis, unit tests
+- Add docs generation using Sphinx, just uses README now, API docs yet to add
+- Move luigi pipeline stuff to cyclops.workflow (yet to be integrated, so
+this feature is broken ATM.
+- Add basic SQL querier, works for admin+diagnosis data
+- Move data-pipeline utils functions to cyclops.processors, clean up some
+functions, and add tests for them.
+- Add logging, wherever it makes sense. Freeze requirements.
+
+### Changed
+- Split config files, so it becomes easier to manage.
+- Move model implementation, and training/validation specific scripts to models.

CONTRIBUTING.md

@@ -0,0 +1,31 @@
+# Contributing to cyclops
+
+Thanks for your interest in contributing to cyclops!
+
+To submit PRs, please fill out the PR template along with the PR. If the PR
+fixes an issue, don't forget to link the PR to the issue!
+
+## Pre-commit hooks
+
+Once the python virtual environment is setup, you can run pre-commit hooks using:
+
+```bash
+pre-commit run --all-files
+```
+
+## Coding guidelines
+
+For code style, we recommend the [google style guide](https://google.github.io/styleguide/pyguide.html).
+
+Pre-commit hooks apply the [black](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html)
+code formatting.
+
+For docstrings we use [numpy format](https://numpydoc.readthedocs.io/en/latest/format.html).
+
+We also use [flake8](https://flake8.pycqa.org/en/latest/) and [pylint](https://pylint.pycqa.org/en/stable/)
+for further static code analysis. The pre-commit hooks show errors which you need
+to fix before submitting a PR.
+
+Last but not the least, we use type hints in our code which is then checked using
+[mypy](https://mypy.readthedocs.io/en/stable/). Currently, mypy checks are not
+strict, but will be enforced more as the API code becomes more stable.

Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.9.7
+
+
+WORKDIR /app/cyclops
+ARG DEBIAN_FRONTEND=noninteractive
+ENV LANG C.UTF-8
+
+
+RUN apt-get update \
+    && apt-get install -y git software-properties-common \
+    && rm -rf /var/lib/apt/lists/*
+
+
+RUN python3 -m pip install --upgrade pip
+RUN pip install poetry
+
+
+COPY * /app/cyclops/
+RUN poetry config virtualenvs.create false \
+   && poetry install --no-interaction --no-ansi