Add integration tests and make a distinction between public and private tests in Github CI

Author: benoit74

.github/workflows/Tests-Private.yaml

@@ -0,0 +1,50 @@
+# This workflow runs everything which is requiring secrets / access to external systems
+# It requires manual approval by a maintainer for external commiters and uses
+# It uses a "hack" to still execute within PR codebase despite secrets being exposed
+name: Private Tests
+
+on:
+  pull_request:
+  pull_request_target:
+  push:
+    branches:
+      - main
+
+jobs:
+  run-tests:
+    strategy:
+      matrix:
+        os: [ubuntu-24.04]
+        python: ["3.11", "3.12"]
+    runs-on: ${{ matrix.os }}
+    environment: 'private'
+    env:
+      A_SECRET_VALUE_FROM_ENV: ${{ secrets.A_SECRET_VALUE_FROM_ENV }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # /!\ important: this checks out code from the HEAD of the PR instead of the main branch (for pull_request_target)
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+
+      - name: Install dependencies (and project)
+        run: |
+          pip install -U pip
+          pip install -e .[test,scripts]
+
+      - name: Run the tests
+        run: inv coverage-integration --args "-vvv"
+
+      - name: Upload coverage results for integration tests
+        if: matrix.python == '3.12'
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-integration
+          path: coverage.xml
+          retention-days: 1

.github/workflows/Tests.yaml .github/workflows/Tests-Public.yaml.github/workflows/Tests.yaml renamed to .github/workflows/Tests-Public.yaml

@@ -1,4 +1,6 @@
-name: Tests
+# This workflow runs everything which is mostly harmless to run on an external PR
+# because it does not requires secrets / access to external systems
+name: Public Tests
 
 on:
   pull_request:
@@ -28,15 +30,31 @@ jobs:
           pip install -U pip
           pip install -e .[test,scripts]
 
-      - name: Run the tests
-        run: inv coverage --args "-vvv"
+      - name: Run the tests with coverage
+        if: matrix.python == '3.12'
+        run: |
+          inv test-unit-cov --args "-vvv"
+          mkdir -p unit-tests
+          cp .coverage* unit-tests
 
-      - name: Upload coverage report to codecov
+      - name: Run the tests without coverage
+        if: matrix.python != '3.12'
+        run: inv test-unit --args "-vvv"
+
+      - name: Upload coverage results for unit tests
         if: matrix.python == '3.12'
-        uses: codecov/codecov-action@v4
+        uses: actions/upload-artifact@v4
         with:
-          fail_ci_if_error: true
-          token: ${{ secrets.CODECOV_TOKEN }}
+          name: coverage-unit
+          path: unit-tests/
+          retention-days: 1
+
+      # - name: Upload coverage report to codecov
+      #   if: matrix.python == '3.12'
+      #   uses: codecov/codecov-action@v4
+      #   with:
+      #     fail_ci_if_error: true
+      #     token: ${{ secrets.CODECOV_TOKEN }}
 
   build_python:
     runs-on: ubuntu-24.04

src/great_project/__init__.py

@@ -1,5 +1,7 @@
 # pyright: strict, reportUnnecessaryIsInstance=false
 
+import os
+
 from great_project.__about__ import __version__
 
 
@@ -10,5 +12,9 @@ def compute(a: int, b: int) -> int:
     return a + b
 
 
+def get_env_value(env_variable: str) -> str | None:
+    return os.environ.get(env_variable)
+
+
 def entrypoint():
     print(f"Hello from {__version__}")  # noqa: T201

tasks.py

@@ -13,12 +13,36 @@ def test(ctx: Context, args: str = ""):
     ctx.run(f"pytest {args}", pty=use_pty)
 
 
+@task(optional=["args"], help={"args": "pytest additional arguments"})
+def test_unit(ctx: Context, args: str = ""):
+    """run unit tests (without coverage)"""
+    ctx.run(f"pytest tests/unit {args}", pty=use_pty)
+
+
+@task(optional=["args"], help={"args": "pytest additional arguments"})
+def test_integration(ctx: Context, args: str = ""):
+    """run integration tests (without coverage)"""
+    ctx.run(f"pytest tests/integration {args}", pty=use_pty)
+
+
 @task(optional=["args"], help={"args": "pytest additional arguments"})
 def test_cov(ctx: Context, args: str = ""):
     """run test vith coverage"""
     ctx.run(f"coverage run -m pytest {args}", pty=use_pty)
 
 
+@task(optional=["args"], help={"args": "pytest additional arguments"})
+def test_unit_cov(ctx: Context, args: str = ""):
+    """run test vith coverage"""
+    ctx.run(f"coverage run -m pytest tests/unit {args}", pty=use_pty)
+
+
+@task(optional=["args"], help={"args": "pytest additional arguments"})
+def test_integration_cov(ctx: Context, args: str = ""):
+    """run test vith coverage"""
+    ctx.run(f"coverage run -m pytest tests/integration {args}", pty=use_pty)
+
+
 @task(optional=["html"], help={"html": "flag to export html report"})
 def report_cov(ctx: Context, *, html: bool = False):
     """report coverage"""
@@ -42,6 +66,32 @@ def coverage(ctx: Context, args: str = "", *, html: bool = False):
     report_cov(ctx, html=html)
 
 
+@task(
+    optional=["args", "html"],
+    help={
+        "args": "pytest additional arguments",
+        "html": "flag to export html report",
+    },
+)
+def coverage_unit(ctx: Context, args: str = "", *, html: bool = False):
+    """run unit tests and report coverage"""
+    test_unit_cov(ctx, args=args)
+    report_cov(ctx, html=html)
+
+
+@task(
+    optional=["args", "html"],
+    help={
+        "args": "pytest additional arguments",
+        "html": "flag to export html report",
+    },
+)
+def coverage_integration(ctx: Context, args: str = "", *, html: bool = False):
+    """run integration tests and report coverage"""
+    test_integration_cov(ctx, args=args)
+    report_cov(ctx, html=html)
+
+
 @task(optional=["args"], help={"args": "black additional arguments"})
 def lint_black(ctx: Context, args: str = "."):
     args = args or "."  # needed for hatch script

tests/integration/test_integration.py

@@ -0,0 +1,8 @@
+# pyright: strict, reportUnusedExpression=false
+
+from great_project import get_env_value
+
+
+# a fake test needing a value from environment variable secret to complete properly
+def test_environ():
+    assert get_env_value("A_SECRET_VALUE_FROM_ENV") == "A_SECRET_VALUE_FROM_ENV"