Merge pull request #186 from crocs-muni/refactor/better-requirements

Refactor/better requirements

.github/workflows/lint.yml

@@ -14,13 +14,9 @@ jobs:
       - name: Install external dependencies
         run: sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y
       - name: Install python dependencies
-        run: pip install -r requirements.txt
-      - name: Install mypy and types
         run: |
-          pip install mypy
-          python3 -m pip install types-PyYAML
-          python3 -m pip install types-python-dateutil
-          python3 -m pip install types-requests
+          pip install -r requirements/requirements.txt
+          pip install -r requirements/dev_requirements.txt
       - name: Run mypy
         run: mypy .
   black:
@@ -37,7 +33,7 @@ jobs:
           python-version: 3.8
       - uses: isort/isort-action@master
         with:
-            requirementsFiles: "requirements.txt dev_requirements.txt"
+            requirementsFiles: "requirements/requirements.txt requirements/dev_requirements.txt"
   flake8-lint:
     runs-on: ubuntu-latest
     name: Flake8

.github/workflows/tests.yml

@@ -5,7 +5,6 @@ on:
 env:
   DEBIAN_FRONTEND: noninteractive
 
-
 jobs:
   run-test:
     runs-on: ubuntu-latest
@@ -24,11 +23,9 @@ jobs:
         with:
           python-version: '3.8'
       - name: Install python dependencies
-        run: pip install -r requirements.txt
+        run: pip install -r requirements/requirements.txt
       - name: Install pytest and package
         run: |
-          pip install pytest
-          pip install pytest-cov
-          pip install ".[dev,test]"
+          pip install -r requirements/test_requirements.txt
       - name: Run tests
         run: pytest --cov=sec_certs tests

CONTRIBUTING.md

@@ -16,7 +16,14 @@ Our [Dockerfile](https://github.com/crocs-muni/sec-certs/blob/main/docker/Docker
 - Some imported libraries have non-trivial dependencies to resolve:
     - [pdftotext](https://github.com/jalan/pdftotext) requires [Poppler](https://poppler.freedesktop.org/) to be installed. We've experienced issues with older versions of Poppler (`0.x`), make sure to install `20.x` version of these libraries.
     - [graphviz](https://pypi.org/project/graphviz/) requires `graphviz` to be on the path
--
+
+### Requirements
+
+Requirements are maintained with [pip-tools](https://github.com/jazzband/pip-tools). The main ideas are:
+- List actual dependencies in `.in` files inside [requirements](requirements) folder without pinning them.
+- Those dependencies are loaded into [setup.py](setup.py) file.
+- Additionally, [compile.sh](requirements/compile.sh) script is used to compile pinned versions of requirements that reside in `.txt` files in the same folder.
+- Tests, linting and Docker all run against this reproducible environment of pinned requirements.
 
 ## Branches and releases
 

docker/Dockerfile

@@ -42,7 +42,7 @@ ENV PATH="${VENV_PATH}/bin:$PATH"
 RUN \
     pip3 install -U pip && \
     pip3 install wheel && \
-    pip3 install -r requirements.txt && \
+    pip3 install -r requirements/requirements.txt && \
     pip3 install --no-cache notebook
 
 # #just to be sure that pdftotext is in $PATH

requirements/compile.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# See CONTRIBUTING.md for description
+
+pip-compile requirements.in --no-header --annotation-style line
+pip-compile dev_requirements.in --no-header --annotation-style line
+pip-compile test_requirements.in --no-header --annotation-style line

requirements/dev_requirements.in

@@ -0,0 +1,13 @@
+mypy
+types-PyYAML
+types-python-dateutil
+types-requests
+pytest
+pytest-cov
+pytest-monitor
+pytest-profiling
+black
+isort
+flake8
+pre-commit
+pip-tools

requirements/dev_requirements.txt

@@ -0,0 +1,53 @@
+attrs==21.4.0             # via pytest
+black==22.3.0             # via -r dev_requirements.in
+certifi==2021.10.8        # via requests
+cfgv==3.3.1               # via pre-commit
+charset-normalizer==2.0.12  # via requests
+click==8.1.2              # via black, pip-tools
+coverage[toml]==6.3.2     # via pytest-cov
+distlib==0.3.4            # via virtualenv
+filelock==3.6.0           # via virtualenv
+flake8==4.0.1             # via -r dev_requirements.in
+gprof2dot==2021.2.21      # via pytest-profiling
+identify==2.4.12          # via pre-commit
+idna==3.3                 # via requests
+iniconfig==1.1.1          # via pytest
+isort==5.10.1             # via -r dev_requirements.in
+mccabe==0.6.1             # via flake8
+memory-profiler==0.60.0   # via pytest-monitor
+mypy==0.942               # via -r dev_requirements.in
+mypy-extensions==0.4.3    # via black, mypy
+nodeenv==1.6.0            # via pre-commit
+packaging==21.3           # via pytest
+pathspec==0.9.0           # via black
+pep517==0.12.0            # via pip-tools
+pip-tools==6.5.1          # via -r dev_requirements.in
+platformdirs==2.5.1       # via black, virtualenv
+pluggy==1.0.0             # via pytest
+pre-commit==2.18.1        # via -r dev_requirements.in
+psutil==5.9.0             # via memory-profiler, pytest-monitor
+py==1.11.0                # via pytest
+pycodestyle==2.8.0        # via flake8
+pyflakes==2.4.0           # via flake8
+pyparsing==3.0.7          # via packaging
+pytest==7.1.1             # via -r dev_requirements.in, pytest-cov, pytest-monitor, pytest-profiling
+pytest-cov==3.0.0         # via -r dev_requirements.in
+pytest-monitor==1.6.3     # via -r dev_requirements.in
+pytest-profiling==1.7.0   # via -r dev_requirements.in
+pyyaml==6.0               # via pre-commit
+requests==2.27.1          # via pytest-monitor
+six==1.16.0               # via pytest-profiling, virtualenv
+toml==0.10.2              # via pre-commit
+tomli==2.0.1              # via black, coverage, mypy, pep517, pytest
+types-python-dateutil==2.8.10  # via -r dev_requirements.in
+types-pyyaml==6.0.5       # via -r dev_requirements.in
+types-requests==2.27.16   # via -r dev_requirements.in
+types-urllib3==1.26.11    # via types-requests
+typing-extensions==4.1.1  # via black, mypy
+urllib3==1.26.9           # via requests
+virtualenv==20.14.0       # via pre-commit
+wheel==0.37.1             # via pip-tools, pytest-monitor
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools

requirements/requirements.in

@@ -0,0 +1,21 @@
+beautifulsoup4
+billiard
+click
+graphviz
+html5lib
+jsonschema
+lxml
+matplotlib
+numpy
+pandas
+pdftotext
+pikepdf
+Pillow
+PyPDF2
+python-dateutil
+PyYAML
+rapidfuzz
+requests
+scikit-learn
+tabula-py
+tqdm

requirements/requirements.txt

@@ -0,0 +1,43 @@
+attrs==21.4.0             # via jsonschema
+beautifulsoup4==4.10.0    # via -r requirements.in
+billiard==3.6.4.0         # via -r requirements.in
+certifi==2021.10.8        # via requests
+charset-normalizer==2.0.12  # via requests
+click==8.1.2              # via -r requirements.in
+cycler==0.11.0            # via matplotlib
+distro==1.7.0             # via tabula-py
+fonttools==4.31.2         # via matplotlib
+graphviz==0.19.1          # via -r requirements.in
+html5lib==1.1             # via -r requirements.in
+idna==3.3                 # via requests
+importlib-resources==5.6.0  # via jsonschema
+jarowinkler==1.0.2        # via rapidfuzz
+joblib==1.1.0             # via scikit-learn
+jsonschema==4.4.0         # via -r requirements.in
+kiwisolver==1.4.2         # via matplotlib
+lxml==4.8.0               # via -r requirements.in, pikepdf
+matplotlib==3.5.1         # via -r requirements.in
+numpy==1.22.3             # via -r requirements.in, matplotlib, pandas, scikit-learn, scipy, tabula-py
+packaging==21.3           # via matplotlib, pikepdf
+pandas==1.4.2             # via -r requirements.in, tabula-py
+pdftotext==2.2.2          # via -r requirements.in
+pikepdf==5.1.1            # via -r requirements.in
+pillow==9.1.0             # via -r requirements.in, matplotlib, pikepdf
+pyparsing==3.0.7          # via matplotlib, packaging
+pypdf2==1.26.0            # via -r requirements.in
+pyrsistent==0.18.1        # via jsonschema
+python-dateutil==2.8.2    # via -r requirements.in, matplotlib, pandas
+pytz==2022.1              # via pandas
+pyyaml==6.0               # via -r requirements.in
+rapidfuzz==2.0.7          # via -r requirements.in
+requests==2.27.1          # via -r requirements.in
+scikit-learn==1.0.2       # via -r requirements.in
+scipy==1.8.0              # via scikit-learn
+six==1.16.0               # via html5lib, python-dateutil
+soupsieve==2.3.1          # via beautifulsoup4
+tabula-py==2.3.0          # via -r requirements.in
+threadpoolctl==3.1.0      # via scikit-learn
+tqdm==4.64.0              # via -r requirements.in
+urllib3==1.26.9           # via requests
+webencodings==0.5.1       # via html5lib
+zipp==3.8.0               # via importlib-resources

requirements/test_requirements.in

@@ -0,0 +1,3 @@
+pytest
+coverage
+pytest-cov

requirements/test_requirements.txt

@@ -0,0 +1,10 @@
+attrs==21.4.0             # via pytest
+coverage[toml]==6.3.2     # via -r test_requirements.in, pytest-cov
+iniconfig==1.1.1          # via pytest
+packaging==21.3           # via pytest
+pluggy==1.0.0             # via pytest
+py==1.11.0                # via pytest
+pyparsing==3.0.7          # via packaging
+pytest==7.1.1             # via -r test_requirements.in, pytest-cov
+pytest-cov==3.0.0         # via -r test_requirements.in
+tomli==2.0.1              # via coverage, pytest

setup.py

@@ -1,12 +1,9 @@
 #!/usr/bin/env python3
 from setuptools import find_packages, setup
 
-with open("requirements.txt") as f:
-    requirements = f.read().splitlines()
-
 setup(
     name="sec-certs",
-    author="Petr Svenda, Stanislav Bobon, Jan Jancar, Adam Janovsky",
+    author="Petr Svenda, Stanislav Bobon, Jan Jancar, Adam Janovsky, Jiri Michalik",
     author_email="svenda@fi.muni.cz",
     version_config=True,
     setup_requires=["setuptools-git-versioning"],
@@ -27,8 +24,11 @@
         "Intended Audience :: Science/Research",
     ],
     python_requires=">=3.8",
-    install_requires=requirements,
-    extras_require={"dev": ["mypy", "flake8"], "test": ["pytest", "coverage"]},
+    install_requires=open("requirements/requirements.in", "r").read().splitlines(),
+    extras_require={
+        "dev": open("requirements/dev_requirements.in", "r").read().splitlines(),
+        "test": open("requirements/test_requirements.in", "r").read().splitlines(),
+    },
     include_package_data=True,
     package_data={"sec_certs": ["settings.yaml", "settings-schema.json"]},
     entry_points={"console_scripts": ["cc-certs=cc_cli:main", "fips-certs=fips_cli:main"]},