Integration Tests for Borg (#1716)

Move existing tests into subfolder `tests/unit`. Write integration tests that actually run the installed borg executable. Those tests can be found in `tests/integration`. Those pytest fixtures that are the same for both kinds of tests remain in `tests/conftest.py`. The others can be found in `tests/integration/conftest.py` or `tests/unit/conftest.py`. This adds nox to the project and configures it to run the tests with different borg versions. This also updates the ci workflow to run the integration tests using nox. * noxfile.py : Run pytest with a matrix of borg versions OR a specific borg version * Makefile : Run using nox. Add phonies `test-unit` and `test-integration`. * tests/conftest.py : Move some fixtures/functions to `tests/unit/conftest.py`. * tests/test_*.py --> tests/unit/ : Move unittests and assets into subfolder * tests/integration/ : Write integration tests. * requirements.d/dev.txt: Add `nox` and `pkgconfig`. The latter is needed for installing new borg versions. * .github/actions/setup/action.yml : Update to install pre-commit and nox when needed. The action now no longer installs Vorta. * .github/actions/install-dependencies/action.yml : Install system deps of borg with this new composite action. * .github/workflows/test.yml : Rename `test` ci to `test-unit` and update it for the new test setup. Implement `test-integration` ci. Signed-off-by: Chirag Aggarwal <thechiragaggarwal@gmail.com>
borgbase · Aug 5, 2023 · b015368 · b015368
1 parent 0e37e1c
commit b015368
Show file tree

Hide file tree

Showing 63 changed files with 1,253 additions and 124 deletions.
diff --git a/.github/actions/install-dependencies/action.yml b/.github/actions/install-dependencies/action.yml
@@ -0,0 +1,22 @@
+name: Install Dependencies
+description: Installs system dependencies
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install system dependencies (Linux)
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        sudo apt update && sudo apt install -y \
+            xvfb libssl-dev openssl libacl1-dev libacl1 fuse3 build-essential \
+            libxkbcommon-x11-0 dbus-x11 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 \
+            libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0 libxcb-shape0 \
+            libegl1 libxcb-cursor0 libfuse-dev libsqlite3-dev libfuse3-dev pkg-config \
+            python3-pkgconfig libxxhash-dev borgbackup
+
+    - name: Install system dependencies (macOS)
+      if: runner.os == 'macOS'
+      shell: bash
+      run: |
+        brew install openssl readline xz xxhash pkg-config borgbackup
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
@@ -15,7 +15,10 @@ inputs:
     description: The python version to install
     required: true
     default: "3.10"
-
+  install-nox:
+    description: Whether nox shall be installed
+    required: false
+    default: "" # == false
 runs:
   using: "composite"
   steps:
@@ -37,16 +40,20 @@ runs:
         restore-keys: |
           ${{ runner.os }}-pip-
 
-    - name: Install Vorta
+    - name: Install pre-commit
       shell: bash
-      run: |
-        pip install -e .
-        pip install -r requirements.d/dev.txt
+      run: pip install pre-commit
+
+    - name: Install nox
+      if: ${{ inputs.install-nox }}
+      shell: bash
+      run: pip install nox
 
     - name: Hash python version
       if: ${{ inputs.setup-pre-commit }}
       shell: bash
       run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
+
     - name: Caching for Pre-Commit
       if: ${{ inputs.setup-pre-commit }}
       uses: actions/cache@v3

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -26,7 +26,7 @@ jobs:
         shell: bash
         run: make lint
 
-  test:
+  test-unit:
     timeout-minutes: 20
     runs-on: ${{ matrix.os }}
     strategy:
@@ -35,40 +35,92 @@ jobs:
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11"]
         os: [ubuntu-latest, macos-latest]
+        borg-version: ["1.2.4"]
 
     steps:
       - uses: actions/checkout@v3
 
-      - name: Install system dependencies (Linux)
+      - name: Install system dependencies
+        uses: ./.github/actions/install-dependencies
+
+      - name: Setup python, vorta and dev deps
+        uses: ./.github/actions/setup
+        with:
+          python-version: ${{ matrix.python-version }}
+          install-nox: true
+
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
+
+      - name: Run Unit Tests with pytest (Linux)
         if: runner.os == 'Linux'
+        env:
+          BORG_VERSION: ${{ matrix.borg-version }}
         run: |
-          sudo apt update && sudo apt install -y \
-            xvfb libssl-dev openssl libacl1-dev libacl1 build-essential borgbackup \
-            libxkbcommon-x11-0 dbus-x11 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 \
-            libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0 libxcb-shape0 \
-            libegl1 libxcb-cursor0
-      - name: Install system dependencies (macOS)
+          xvfb-run --server-args="-screen 0 1024x768x24+32" \
+                    -a dbus-run-session -- make test-unit
+
+      - name: Run Unit Tests with pytest (macOS)
         if: runner.os == 'macOS'
-        run: |
-          brew install openssl readline xz borgbackup
+        env:
+          BORG_VERSION: ${{ matrix.borg-version }}
+          PKG_CONFIG_PATH: /usr/local/opt/openssl@3/lib/pkgconfig
+        run: echo $PKG_CONFIG_PATH && make test-unit
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        env:
+          OS: ${{ runner.os }}
+          python: ${{ matrix.python-version }}
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          env_vars: OS, python
+
+  test-integration:
+    timeout-minutes: 20
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        os: [ubuntu-latest, macos-latest]
+        borg-version: ["1.1.18", "1.2.2", "1.2.4", "2.0.0b5"]
+        exclude:
+          - borg-version: "2.0.0b5"
+            python-version: "3.8"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install system dependencies
+        uses: ./.github/actions/install-dependencies
 
       - name: Setup python, vorta and dev deps
         uses: ./.github/actions/setup
         with:
           python-version: ${{ matrix.python-version }}
+          install-nox: true
 
       - name: Setup tmate session
         uses: mxschmitt/action-tmate@v3
         if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
 
-      - name: Test with pytest (Linux)
+      - name: Run Integration Tests with pytest (Linux)
         if: runner.os == 'Linux'
+        env:
+          BORG_VERSION: ${{ matrix.borg-version }}
         run: |
           xvfb-run --server-args="-screen 0 1024x768x24+32" \
-                   -a dbus-run-session -- make test
-      - name: Test with pytest (macOS)
+                   -a dbus-run-session -- make test-integration
+
+      - name: Run Integration Tests with pytest (macOS)
         if: runner.os == 'macOS'
-        run: make test
+        env:
+          BORG_VERSION: ${{ matrix.borg-version }}
+          PKG_CONFIG_PATH: /usr/local/opt/openssl@3/lib/pkgconfig
+        run: echo $PKG_CONFIG_PATH && make test-integration
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3

diff --git a/Makefile b/Makefile
@@ -67,7 +67,13 @@ lint:
 	pre-commit run --all-files --show-diff-on-failure
 
 test:
-	pytest --cov=vorta
+	nox -- --cov=vorta
+
+test-unit:
+	nox -- --cov=vorta tests/unit
+
+test-integration:
+	nox -- --cov=vorta tests/integration
 
 help:
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
diff --git a/noxfile.py b/noxfile.py
@@ -0,0 +1,56 @@
+import os
+import re
+import sys
+
+import nox
+
+borg_version = os.getenv("BORG_VERSION")
+
+if borg_version:
+    # Use specified borg version
+    supported_borgbackup_versions = [borg_version]
+else:
+    # Generate a list of borg versions compatible with system installed python version
+    system_python_version = tuple(sys.version_info[:3])
+
+    supported_borgbackup_versions = [
+        borgbackup
+        for borgbackup in ("1.1.18", "1.2.2", "1.2.4", "2.0.0b6")
+        # Python version requirements for borgbackup versions
+        if (borgbackup == "1.1.18" and system_python_version >= (3, 5, 0))
+        or (borgbackup == "1.2.2" and system_python_version >= (3, 8, 0))
+        or (borgbackup == "1.2.4" and system_python_version >= (3, 8, 0))
+        or (borgbackup == "2.0.0b6" and system_python_version >= (3, 9, 0))
+    ]
+
+
+@nox.session
+@nox.parametrize("borgbackup", supported_borgbackup_versions)
+def run_tests(session, borgbackup):
+    # install borgbackup
+    if (sys.platform == 'darwin'):
+        # in macOS there's currently no fuse package which works with borgbackup directly
+        session.install(f"borgbackup=={borgbackup}")
+    elif (borgbackup == "1.1.18"):
+        # borgbackup 1.1.18 doesn't support pyfuse3
+        session.install("llfuse")
+        session.install(f"borgbackup[llfuse]=={borgbackup}")
+    else:
+        session.install(f"borgbackup[pyfuse3]=={borgbackup}")
+
+    # install dependencies
+    session.install("-r", "requirements.d/dev.txt")
+    session.install("-e", ".")
+
+    # check versions
+    cli_version = session.run("borg", "--version", silent=True).strip()
+    cli_version = re.search(r"borg (\S+)", cli_version).group(1)
+    python_version = session.run("python", "-c", "import borg; print(borg.__version__)", silent=True).strip()
+
+    session.log(f"Borg CLI version: {cli_version}")
+    session.log(f"Borg Python version: {python_version}")
+
+    assert cli_version == borgbackup
+    assert python_version == borgbackup
+
+    session.run("pytest", *session.posargs, env={"BORG_VERSION": borgbackup})
diff --git a/requirements.d/dev.txt b/requirements.d/dev.txt
@@ -2,6 +2,8 @@ black==22.*
 coverage
 flake8
 macholib
+nox
+pkgconfig
 pre-commit
 pyinstaller
 pylint

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,37 +1,11 @@
 import os
 import sys
-from datetime import datetime as dt
-from unittest.mock import MagicMock
 
 import pytest
 import vorta
 import vorta.application
 import vorta.borg.jobs_manager
 from peewee import SqliteDatabase
-from vorta.store.models import (
-    ArchiveModel,
-    BackupProfileModel,
-    EventLogModel,
-    RepoModel,
-    RepoPassword,
-    SchemaVersion,
-    SettingsModel,
-    SourceFileModel,
-    WifiSettingModel,
-)
-from vorta.views.main_window import MainWindow
-
-models = [
-    RepoModel,
-    RepoPassword,
-    BackupProfileModel,
-    SourceFileModel,
-    SettingsModel,
-    ArchiveModel,
-    WifiSettingModel,
-    EventLogModel,
-    SchemaVersion,
-]
 
 
 def pytest_configure(config):
@@ -55,86 +29,8 @@ def qapp(tmpdir_factory):
 
     from vorta.application import VortaApp
 
-    VortaApp.set_borg_details_action = MagicMock()  # Can't use pytest-mock in session scope
-    VortaApp.scheduler = MagicMock()
-
     qapp = VortaApp([])  # Only init QApplication once to avoid segfaults while testing.
 
     yield qapp
     mock_db.close()
     qapp.quit()
-
-
-@pytest.fixture(scope='function', autouse=True)
-def init_db(qapp, qtbot, tmpdir_factory):
-    tmp_db = tmpdir_factory.mktemp('Vorta').join('settings.sqlite')
-    mock_db = SqliteDatabase(
-        str(tmp_db),
-        pragmas={
-            'journal_mode': 'wal',
-        },
-    )
-    vorta.store.connection.init_db(mock_db)
-
-    default_profile = BackupProfileModel(name='Default')
-    default_profile.save()
-
-    new_repo = RepoModel(url='i0fi93@i593.repo.borgbase.com:repo')
-    new_repo.encryption = 'none'
-    new_repo.save()
-
-    default_profile.repo = new_repo.id
-    default_profile.dont_run_on_metered_networks = False
-    default_profile.validation_on = False
-    default_profile.save()
-
-    test_archive = ArchiveModel(snapshot_id='99999', name='test-archive', time=dt(2000, 1, 1, 0, 0), repo=1)
-    test_archive.save()
-
-    test_archive1 = ArchiveModel(snapshot_id='99998', name='test-archive1', time=dt(2000, 1, 1, 0, 0), repo=1)
-    test_archive1.save()
-
-    source_dir = SourceFileModel(dir='/tmp/another', repo=new_repo, dir_size=100, dir_files_count=18, path_isdir=True)
-    source_dir.save()
-
-    qapp.main_window.deleteLater()
-    del qapp.main_window
-    qapp.main_window = MainWindow(qapp)  # Re-open main window to apply mock data in UI
-
-    yield
-
-    qapp.jobs_manager.cancel_all_jobs()
-    qapp.backup_finished_event.disconnect()
-    qapp.scheduler.schedule_changed.disconnect()
-    qtbot.waitUntil(lambda: not qapp.jobs_manager.is_worker_running(), **pytest._wait_defaults)
-    mock_db.close()
-
-
-@pytest.fixture
-def choose_file_dialog(*args):
-    class MockFileDialog:
-        def __init__(self, *args, **kwargs):
-            pass
-
-        def open(self, func):
-            func()
-
-        def selectedFiles(self):
-            return ['/tmp']
-
-    return MockFileDialog
-
-
-@pytest.fixture
-def borg_json_output():
-    def _read_json(subcommand):
-        stdout = open(f'tests/borg_json_output/{subcommand}_stdout.json')
-        stderr = open(f'tests/borg_json_output/{subcommand}_stderr.json')
-        return stdout, stderr
-
-    return _read_json
-
-
-@pytest.fixture
-def rootdir():
-    return os.path.dirname(os.path.abspath(__file__))
diff --git a/tests/borg_json_output/compact_stdout.json → tests/integration/__init__.py b/tests/borg_json_output/compact_stdout.json → tests/integration/__init__.py