From 9cee1d354469cf1f02c98c70e96d788c165b75e2 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 15 Aug 2023 20:42:56 +0200 Subject: [PATCH 01/17] WIP --- .gitignore | 1 + tests/Makefile | 14 ++++++++++---- tests/external-test-files.list | 1 + tests/tests_selenium.py | 30 ++++++++++++++++++++++++++++++ update.sh | 1 + 5 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 tests/external-test-files.list diff --git a/.gitignore b/.gitignore index 9873055..a04f5e3 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ __pycache__ /tests/latest_logs /tests/report.html /tests/assets/style.css +/tests/external-test-files diff --git a/tests/Makefile b/tests/Makefile index 7fe4aa8..47db79f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,13 +1,19 @@ - -test: .env/bin/seleniumbase wait +test: .env/bin/seleniumbase external-test-files wait .env/bin/pytest tests_selenium.py --server=127.0.0.1 --port=4444 --headed --html=report.html -trace: .env/bin/seleniumbase wait +trace: .env/bin/seleniumbase external-test-files wait .env/bin/pytest tests_selenium.py --server=127.0.0.1 --port=4444 --headed --html=report.html --trace -ci: wait +ci: external-test-files wait pytest tests_selenium.py --server=127.0.0.1 --port=4444 --html=report.html +external-test-files: + mkdir -p external-test-files + + cd external-test-files && for file in $(shell cat external-test-files.list); do \ + wget $$file; \ + done + wait: while ! ./test_haproxy.sh ; do sleep 1; done diff --git a/tests/external-test-files.list b/tests/external-test-files.list new file mode 100644 index 0000000..fb411af --- /dev/null +++ b/tests/external-test-files.list @@ -0,0 +1 @@ +http://www.rawsamples.ch/raws/sony/a100/RAW_SONY_A100.ARW diff --git a/tests/tests_selenium.py b/tests/tests_selenium.py index b816aca..0e1d8f6 100755 --- a/tests/tests_selenium.py +++ b/tests/tests_selenium.py @@ -163,6 +163,36 @@ def test_005_upload_assets_with_cli(self): # Give the system time to process the new assets self.sleep(60) + def test_005_upload_external_assets_with_cli(self): + """ + Use the CLI to upload assets from the external-test-files directory. + """ + secret = get_secret() + + snap_readable_path = os.path.join( + os.environ["HOME"], + "snap/immich-distribution/current/tests_external" + ) + + if not os.path.exists(snap_readable_path): + os.makedirs(snap_readable_path) + + for upload in os.listdir("external-test-files"): + shutil.copy(f"external-test-files/{upload}", snap_readable_path) + + subprocess.run( + [ + "immich-distribution.cli", + "upload", + "--key", secret, + "--yes", + snap_readable_path + ] + ) + + # Give the system time to process the new assets + self.sleep(60) + def test_100_verify_uploaded_assets_number_of_files(self): """ Use the API to verify that the assets were uploaded correctly. diff --git a/update.sh b/update.sh index 6d07ea0..7f18aba 100755 --- a/update.sh +++ b/update.sh @@ -58,6 +58,7 @@ CHECK_FILES=" machine-learning/Dockerfile docker/example.env docker/docker-compose.yml + docs/docs/install/environment-variables.md " for F in $CHECK_FILES; do From 06dff8c449d6d783236e9004d4d6515779e0bb60 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Sat, 19 Aug 2023 10:39:42 +0200 Subject: [PATCH 02/17] Download testfiles via makefile --- .gitignore | 1 + tests/Makefile | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index a04f5e3..37ebc05 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ __pycache__ /tests/latest_logs /tests/report.html /tests/assets/style.css +/tests/assets/tests_selenium.py__* /tests/external-test-files diff --git a/tests/Makefile b/tests/Makefile index 47db79f..5bb649d 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,12 +1,18 @@ -test: .env/bin/seleniumbase external-test-files wait +test: .env/bin/seleniumbase prepare-test-file wait .env/bin/pytest tests_selenium.py --server=127.0.0.1 --port=4444 --headed --html=report.html -trace: .env/bin/seleniumbase external-test-files wait +trace: .env/bin/seleniumbase prepare-test-file wait .env/bin/pytest tests_selenium.py --server=127.0.0.1 --port=4444 --headed --html=report.html --trace -ci: external-test-files wait +ci: prepare-test-file wait pytest tests_selenium.py --server=127.0.0.1 --port=4444 --html=report.html +prepare-test-file: external-test-files + mkdir -p ${HOME}/snap/immich-distribution/current/tests + mkdir -p ${HOME}/snap/immich-distribution/current/tests_external + cp -r assets ${HOME}/snap/immich-distribution/current/tests + cp -r external-test-files ${HOME}/snap/immich-distribution/current/tests_external + external-test-files: mkdir -p external-test-files From 3cfd71e4e0fecc1b03717d1ea75e71e7b1f77b16 Mon Sep 17 00:00:00 2001 From: Github Action Automation Date: Wed, 6 Sep 2023 06:03:24 +0000 Subject: [PATCH 03/17] Bump version to v1.77.0 --- VERSION | 2 +- parts/machine-learning/Makefile | 2 +- patches/Makefile | 2 +- snap/snapcraft.yaml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index fb868ec..1ed23ab 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.76.1 +v1.77.0 diff --git a/parts/machine-learning/Makefile b/parts/machine-learning/Makefile index 9dc8ecd..94bfbb4 100644 --- a/parts/machine-learning/Makefile +++ b/parts/machine-learning/Makefile @@ -1,6 +1,6 @@ .PHONY: build build: - ./build.sh v1.76.1 + ./build.sh v1.77.0 .PHONY: install install: diff --git a/patches/Makefile b/patches/Makefile index 577b578..76d7c62 100644 --- a/patches/Makefile +++ b/patches/Makefile @@ -1,4 +1,4 @@ -VERSION=v1.76.1 +VERSION=v1.77.0 SOURCE_FILE_PATH=https://raw.githubusercontent.com/immich-app/immich/${VERSION}/web/src/lib/components/shared-components/version-announcement-box.svelte download-patch: diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index c6d3076..2bf88d1 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -182,7 +182,7 @@ parts: plugin: npm npm-node-version: "18.16.0" source: https://github.com/immich-app/immich.git - source-tag: v1.76.1 + source-tag: v1.77.0 source-subdir: server override-build: | snapcraftctl set-version "$(git describe --tags)-dist1" @@ -203,7 +203,7 @@ parts: plugin: npm npm-node-version: "18.16.0" source: https://github.com/immich-app/immich.git - source-tag: v1.76.1 + source-tag: v1.77.0 source-subdir: web override-build: | patch -p0 -i $SNAPCRAFT_PART_SRC/../../patches/src/001-version-announcement-box.patch -d $SNAPCRAFT_PART_BUILD From cff3ae888805bf31f241d24fb9afcc0ee3d7db33 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Thu, 7 Sep 2023 17:07:50 +0200 Subject: [PATCH 04/17] Rewrite build and launch process for v1.77 --- parts/machine-learning/Makefile | 55 ++++++++++++++++--- parts/machine-learning/build.sh | 31 ----------- .../machine-learning/immich-machine-learning | 16 ++++++ snap/snapcraft.yaml | 1 + src/bin/immich-machine-learning | 10 ---- src/bin/load-env | 5 ++ update.sh | 1 + 7 files changed, 71 insertions(+), 48 deletions(-) delete mode 100755 parts/machine-learning/build.sh create mode 100755 parts/machine-learning/immich-machine-learning delete mode 100755 src/bin/immich-machine-learning diff --git a/parts/machine-learning/Makefile b/parts/machine-learning/Makefile index 94bfbb4..d1334d5 100644 --- a/parts/machine-learning/Makefile +++ b/parts/machine-learning/Makefile @@ -1,11 +1,52 @@ +IMMICH_VERSION := v1.77.0 + +# Use our own Python, over the older one in core20 +export PATH := ${SNAPCRAFT_PART_BUILD}/../../dependencies/install/usr/local/bin:$(PATH) + +POETRY_ENV = ${HOME}/poetry +POETRY = ${POETRY_ENV}/bin/poetry + .PHONY: build -build: - ./build.sh v1.77.0 +build: immich ${POETRY_ENV} + ${POETRY} install \ + -C immich/machine-learning \ + --sync \ + --no-interaction \ + --no-ansi \ + --no-root \ + --only main + + ${POETRY} run \ + -C immich/machine-learning \ + pip install \ + --no-deps \ + -r immich/machine-learning/requirements.txt + +immich: + git clone --branch ${IMMICH_VERSION} https://github.com/immich-app/immich.git + +${POETRY_ENV}: + python3 -m venv ${POETRY_ENV} + ${POETRY_ENV}/bin/pip install -U pip setuptools + ${POETRY_ENV}/bin/pip install poetry .PHONY: install install: - mkdir -p ${SNAPCRAFT_PART_INSTALL}/usr/src/ml - mkdir -p ${SNAPCRAFT_PART_INSTALL}/opt/python-libs/ml - cp -r immich/machine-learning/app ${SNAPCRAFT_PART_INSTALL}/usr/src/ml - cp -r /opt/venv/ml/lib/python3.11/site-packages \ - ${SNAPCRAFT_PART_INSTALL}/opt/python-libs/ml/ + mkdir -p ${SNAPCRAFT_PART_INSTALL}/app/machine-learning + mkdir -p ${SNAPCRAFT_PART_INSTALL}/bin + + # Deploy application + cp immich/machine-learning/log_conf.json ${SNAPCRAFT_PART_INSTALL}/app/machine-learning/ + cp -r immich/machine-learning/app ${SNAPCRAFT_PART_INSTALL}/app/machine-learning/app + + # Deploy Python packages (installed with Poetry & pip in the build step) + cp -r $(shell ${POETRY} -C immich/machine-learning env info -p)/lib/python3.11/site-packages \ + ${SNAPCRAFT_PART_INSTALL}/app/machine-learning/python-packages + + # Deploy gunicorn (installed with Poetry) and remove shebang + cp $(shell ${POETRY} -C immich/machine-learning env info -p)/bin/gunicorn \ + ${SNAPCRAFT_PART_INSTALL}/bin/gunicorn + sed -i '1d' ${SNAPCRAFT_PART_INSTALL}/bin/gunicorn + + # Deploy immich-machine-learning script + cp immich-machine-learning ${SNAPCRAFT_PART_INSTALL}/bin/immich-machine-learning diff --git a/parts/machine-learning/build.sh b/parts/machine-learning/build.sh deleted file mode 100755 index 6e9dbf0..0000000 --- a/parts/machine-learning/build.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -VERSION="$1" -PYTHON_PART_INSTALL_BIN="${PWD}/../../dependencies/install/usr/local/bin" - -mkdir -p /opt/venv/ - -python() { - $PYTHON_PART_INSTALL_BIN/python3 $@ -} - -pip() { - python $PYTHON_PART_INSTALL_BIN/pip3 $@ -} - -[ -d immich ] || git clone --branch $VERSION https://github.com/immich-app/immich.git -[ -d "/opt/venv/ml" ] || python -m venv "/opt/venv/ml" -[ -d "/opt/venv/poetry" ] || python -m venv "/opt/venv/poetry" - -/opt/venv/poetry/bin/python --version -/opt/venv/poetry/bin/pip install poetry - -# Configure Poetry and install dependencies -cd immich/machine-learning - -export VIRTUAL_ENV="/opt/venv/ml" -/opt/venv/poetry/bin/poetry config installer.max-workers 10 -/opt/venv/poetry/bin/poetry config virtualenvs.create false -/opt/venv/poetry/bin/poetry install --sync --no-interaction --no-ansi --no-root --only main -/opt/venv/poetry/bin/poetry run pip install --no-deps -r requirements.txt -unset VIRTUAL_ENV diff --git a/parts/machine-learning/immich-machine-learning b/parts/machine-learning/immich-machine-learning new file mode 100755 index 0000000..cf72fe0 --- /dev/null +++ b/parts/machine-learning/immich-machine-learning @@ -0,0 +1,16 @@ +#!/bin/bash + +. $SNAP/bin/load-env + +export PYTHONPATH="$SNAP/app/machine-learning/python-packages" +export LD_PRELOAD="$SNAP/usr/local/lib/libmimalloc.so.2" + +echo "Launching machine-learning service using python version: $(python3 --version)" +echo "PYTHONPATH: $PYTHONPATH" + +cd $SNAP/app/machine-learning +python3 $SNAP/bin/gunicorn app.main:app \ + -k uvicorn.workers.UvicornWorker \ + -w $MACHINE_LEARNING_WORKERS \ + -b $MACHINE_LEARNING_HOST:$MACHINE_LEARNING_PORT \ + --log-config-json log_conf.json diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 2bf88d1..db0f646 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -255,6 +255,7 @@ parts: - immich-distribution-libvips/latest/stable - immich-distribution-node/latest/stable - immich-distribution-python/latest/stable + - immich-distribution-mimalloc/latest/stable scripts: source: src diff --git a/src/bin/immich-machine-learning b/src/bin/immich-machine-learning deleted file mode 100755 index 1b0af3c..0000000 --- a/src/bin/immich-machine-learning +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -. $SNAP/bin/load-env - -export PYTHONDONTWRITEBYTECODE=1 -export PYTHONUNBUFFERED=1 -export PYTHONPATH="$SNAP/opt/python-libs/ml/site-packages/:$SNAP/usr/src/ml" -export MACHINE_LEARNING_CACHE_FOLDER="$SNAP_COMMON/cache" - -$SNAP/usr/local/bin/python3 -m app.main | grep -v "GET /ping" diff --git a/src/bin/load-env b/src/bin/load-env index 33e2e75..4a39964 100755 --- a/src/bin/load-env +++ b/src/bin/load-env @@ -8,6 +8,7 @@ snapctl() { fi } +export PATH="$SNAP/usr/local/bin:$PATH" export LD_LIBRARY_PATH="$SNAP/usr/local/pgsql/lib:$SNAP/usr/lib/x86_64-linux-gnu/pulseaudio:$SNAP/usr/lib/x86_64-linux-gnu/libfswatch:$SNAP/usr/local/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH" export PGDATA="$SNAP_COMMON/pgsql/data" @@ -32,6 +33,10 @@ export IMMICH_SERVER_ADDRESS="$IMMICH_SERVER_URL" # CLI export IMMICH_MACHINE_LEARNING_URL="http://127.0.0.1:3003" export MACHINE_LEARNING_LOG_LEVEL="warning" export MACHINE_LEARNING_IP="127.0.0.1" +export MACHINE_LEARNING_CACHE_FOLDER="$SNAP_COMMON/cache" +export MACHINE_LEARNING_HOST="0.0.0.0" +export MACHINE_LEARNING_PORT=3003 +export MACHINE_LEARNING_WORKERS=1 export TYPESENSE_API_KEY="$(snapctl get typesense-key)" export TYPESENSE_DATA_DIR="$SNAP_COMMON/typesense" diff --git a/update.sh b/update.sh index 6d07ea0..8e54faf 100755 --- a/update.sh +++ b/update.sh @@ -56,6 +56,7 @@ CHECK_FILES=" nginx machine-learning/README.md machine-learning/Dockerfile + machine-learning/start.sh docker/example.env docker/docker-compose.yml " From 643b651132849de4c313e9efb99169b0e4e89a33 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Thu, 7 Sep 2023 18:22:44 +0200 Subject: [PATCH 05/17] Increase wait time --- tests/tests_selenium.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_selenium.py b/tests/tests_selenium.py index b816aca..1ff1289 100755 --- a/tests/tests_selenium.py +++ b/tests/tests_selenium.py @@ -161,7 +161,7 @@ def test_005_upload_assets_with_cli(self): ) # Give the system time to process the new assets - self.sleep(60) + self.sleep(240) def test_100_verify_uploaded_assets_number_of_files(self): """ From 4497e937c33a0a4bcbecc328942e3a8342a75c56 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Thu, 7 Sep 2023 19:14:04 +0200 Subject: [PATCH 06/17] rule out a race condition --- tests/tests_selenium.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_selenium.py b/tests/tests_selenium.py index 1ff1289..4141dc6 100755 --- a/tests/tests_selenium.py +++ b/tests/tests_selenium.py @@ -161,7 +161,7 @@ def test_005_upload_assets_with_cli(self): ) # Give the system time to process the new assets - self.sleep(240) + self.sleep(300) def test_100_verify_uploaded_assets_number_of_files(self): """ From d7385dec6f0f9f07021f032bee842830bab2939c Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 12 Sep 2023 15:44:59 +0200 Subject: [PATCH 07/17] Add a target to uninstall the snap --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 9d5be9b..7c36f9f 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,10 @@ build: install: sudo snap install --dangerous ${SNAP_FILE} +.PHONY: remove +remove: + sudo snap remove --purge immich-distribution + beta: build cat ${SNAP_FILE} | ssh d -- lxc file push - immich-beta/root/${SNAP_FILE} ssh d lxc exec immich-beta -- snap install --dangerous /root/${SNAP_FILE} From af03c1b0f20b7d7cc75c91977396cb3a7a1409b7 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 12 Sep 2023 15:45:22 +0200 Subject: [PATCH 08/17] Expose tests and selenium from outer Makefile --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index 7c36f9f..ba94a2e 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,14 @@ beta2store: shell: multipass shell snapcraft-immich-distribution +.PHONY: tests +tests: + make -C tests test + +.PHONY: selenium +selenium: + make -C tests/ selenium + .PHONY: docs docs: cd docs && poetry run mkdocs serve From ec05fe4ca56f42ea51c80121047ccd4d0bbb63d2 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 12 Sep 2023 15:45:41 +0200 Subject: [PATCH 09/17] Show LD_PRELOAD from ML startup message --- parts/machine-learning/immich-machine-learning | 1 + 1 file changed, 1 insertion(+) diff --git a/parts/machine-learning/immich-machine-learning b/parts/machine-learning/immich-machine-learning index cf72fe0..ad4ab66 100755 --- a/parts/machine-learning/immich-machine-learning +++ b/parts/machine-learning/immich-machine-learning @@ -7,6 +7,7 @@ export LD_PRELOAD="$SNAP/usr/local/lib/libmimalloc.so.2" echo "Launching machine-learning service using python version: $(python3 --version)" echo "PYTHONPATH: $PYTHONPATH" +echo "LD_PRELOAD: $LD_PRELOAD" cd $SNAP/app/machine-learning python3 $SNAP/bin/gunicorn app.main:app \ From 4b0ffe042c47c40d7a45852f4b39afbf3ea08029 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 12 Sep 2023 15:45:59 +0200 Subject: [PATCH 10/17] Set maxconn to a value lower then ML backend's 8 I set 7 to save one worker thread for health checks, with this I hope that I will queue requests inside HAProxy instead of throwing them down to a overloaded backend. I'm not sure that's a good value for microservices so I picked 7 there as well :) --- src/etc/haproxy.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/etc/haproxy.cfg b/src/etc/haproxy.cfg index 2027e68..f4e3976 100644 --- a/src/etc/haproxy.cfg +++ b/src/etc/haproxy.cfg @@ -54,12 +54,12 @@ backend be_typesense backend be_microservices option httpchk http-check send meth GET uri /ping - server microservices 127.0.0.1:3003 maxconn 32 check inter 10s fall 2 rise 6 + server microservices 127.0.0.1:3003 maxconn 7 check inter 10s fall 2 rise 6 backend be_ml option httpchk http-check send meth GET uri /ping - server ml 127.0.0.1:3003 maxconn 32 check inter 10s fall 2 rise 6 + server ml 127.0.0.1:3003 maxconn 7 check inter 10s fall 2 rise 6 backend be_postgres mode tcp From 5b652030548efe21c1206f00d0b825c4445013d2 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 12 Sep 2023 15:48:18 +0200 Subject: [PATCH 11/17] Check if selenium is running --- tests/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index 54a7fc3..dd409f6 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,4 +1,4 @@ -test: .env/bin/seleniumbase prepare-test-file wait +test: .env/bin/seleniumbase prepare-test-file wait check-selenium .env/bin/pytest tests_selenium.py --server=127.0.0.1 --port=4444 --headed --html=report.html trace: .env/bin/seleniumbase prepare-test-file wait @@ -23,6 +23,9 @@ external-test-files: wait: ./test_haproxy.sh +check-selenium: + nc -v -w1 127.0.0.1 4444 || (echo "Selenium is not running"; exit 1) + selenium: podman run \ -p 4444:4444 \ From fd6450362e3ab37b4c42b53dfb1a728f27545512 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 12 Sep 2023 15:49:16 +0200 Subject: [PATCH 12/17] Make sure that snap's symlinks are there The current symlink is created on first run, so I run the cli just to trigger a creation of symlinks for the current user. --- tests/Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index dd409f6..ec9aaa9 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -7,19 +7,22 @@ trace: .env/bin/seleniumbase prepare-test-file wait ci: prepare-test-file wait pytest tests_selenium.py --server=127.0.0.1 --port=4444 --html=report.html -prepare-test-file: external-test-files +prepare-test-file: external-test-files update-snap-symlinks-for-user mkdir -p ${HOME}/snap/immich-distribution/current/tests mkdir -p ${HOME}/snap/immich-distribution/current/tests_external cp -r assets ${HOME}/snap/immich-distribution/current/tests cp -r external-test-files ${HOME}/snap/immich-distribution/current/tests_external -external-test-files: +external-test-files: update-snap-symlinks-for-user mkdir -p external-test-files cd external-test-files && for file in $(shell cat external-test-files.list); do \ wget $$file; \ done +update-snap-symlinks-for-user: + immich-distribution.cli -h > /dev/null 2>&1 + wait: ./test_haproxy.sh From 1fe4695c99828e2d544bc5778609d2542fcf51e6 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 12 Sep 2023 15:50:46 +0200 Subject: [PATCH 13/17] Ignore assets from pytest --- tests/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index ec9aaa9..bdeb835 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -10,7 +10,10 @@ ci: prepare-test-file wait prepare-test-file: external-test-files update-snap-symlinks-for-user mkdir -p ${HOME}/snap/immich-distribution/current/tests mkdir -p ${HOME}/snap/immich-distribution/current/tests_external - cp -r assets ${HOME}/snap/immich-distribution/current/tests + for asset in assets/*; do \ + echo $$asset | grep -q py__ \ + || cp $$asset ${HOME}/snap/immich-distribution/current/tests/; \ + done cp -r external-test-files ${HOME}/snap/immich-distribution/current/tests_external external-test-files: update-snap-symlinks-for-user From 1db60a146544018d3bc60506ca417c996b89eedb Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 12 Sep 2023 15:51:08 +0200 Subject: [PATCH 14/17] Do not re-download files --- tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index bdeb835..ad0f357 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -20,7 +20,7 @@ external-test-files: update-snap-symlinks-for-user mkdir -p external-test-files cd external-test-files && for file in $(shell cat external-test-files.list); do \ - wget $$file; \ + wget -c $$file; \ done update-snap-symlinks-for-user: From f23b290792b02491875654dcfbd89b4aee3afb06 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 12 Sep 2023 15:51:32 +0200 Subject: [PATCH 15/17] Remove unused volume mount --- tests/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index ad0f357..8c36268 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -37,7 +37,6 @@ selenium: -p 4444:4444 \ -p 7900:7900 \ --shm-size="2g" \ - -v ${PWD}/assets:/assets \ docker.io/selenium/standalone-chrome:latest .env: From 33b07b43d9c984c7de73b913778c1a7a52d29945 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 12 Sep 2023 15:51:45 +0200 Subject: [PATCH 16/17] Remove RAW image, to disable RAW test (for now) --- tests/external-test-files.list | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/external-test-files.list b/tests/external-test-files.list index fb411af..e69de29 100644 --- a/tests/external-test-files.list +++ b/tests/external-test-files.list @@ -1 +0,0 @@ -http://www.rawsamples.ch/raws/sony/a100/RAW_SONY_A100.ARW From f6821c4d86f5d37af8f14e54c0449af6d5b4c72a Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 12 Sep 2023 15:52:12 +0200 Subject: [PATCH 17/17] Check and run queues instead of sleeping This is a much better solution, to actually wait for the processing to finish before we carry on. --- tests/tests_selenium.py | 63 +++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/tests/tests_selenium.py b/tests/tests_selenium.py index 1a33841..a8dcf61 100755 --- a/tests/tests_selenium.py +++ b/tests/tests_selenium.py @@ -5,6 +5,7 @@ import subprocess import shutil import requests +import time from seleniumbase import BaseCase BaseCase.main(__name__, __file__) @@ -37,6 +38,26 @@ def get_assets(filter=[]): return resp +def get_all_jobs(): + r = requests.get(f"http://{get_ip_address()}/api/jobs", headers=get_headers()) + return r.json() + +def trigger_job(job_name): + r = requests.put( + f"http://{get_ip_address()}/api/jobs/{job_name}", + headers=get_headers(), + json={"command": "start"} + ) + return r.json() + +def wait_for_empty_job_queue(): + for job_name, job_data in get_all_jobs().items(): + status = job_data['queueStatus']['isActive'] + if status == True: + print(f"Queue {job_name} is running") + time.sleep(1) + return wait_for_empty_job_queue() + def css_selector_path(element): """ Returns a CSS selector that will uniquely select the given element. """ path = [] @@ -141,57 +162,37 @@ def test_005_upload_assets_with_cli(self): snap_readable_path = os.path.join( os.environ["HOME"], - "snap/immich-distribution/current/tests" + "snap/immich-distribution/current/" ) - if not os.path.exists(snap_readable_path): - os.makedirs(snap_readable_path) - - for upload in os.listdir("assets"): - shutil.copy(f"assets/{upload}", snap_readable_path) - subprocess.run( [ "immich-distribution.cli", "upload", "--key", secret, "--yes", - snap_readable_path + f"{snap_readable_path}/tests" ] ) - # Give the system time to process the new assets - self.sleep(300) - - def test_005_upload_external_assets_with_cli(self): - """ - Use the CLI to upload assets from the external-test-files directory. - """ - secret = get_secret() - - snap_readable_path = os.path.join( - os.environ["HOME"], - "snap/immich-distribution/current/tests_external" - ) - - if not os.path.exists(snap_readable_path): - os.makedirs(snap_readable_path) - - for upload in os.listdir("external-test-files"): - shutil.copy(f"external-test-files/{upload}", snap_readable_path) - subprocess.run( [ "immich-distribution.cli", "upload", "--key", secret, "--yes", - snap_readable_path + f"{snap_readable_path}/tests_external" ] ) - # Give the system time to process the new assets - self.sleep(60) + # ML models are downloaded in the background when we upload assets + # Wait for them to complete, and the queue to be empty before continuing + wait_for_empty_job_queue() + + # Re-run the recognition job. I'm not sure if this is an Immich bug or + # just a quirk of the test environment. Anyway let's just run it again. + trigger_job("recognizeFaces") + wait_for_empty_job_queue() def test_100_verify_uploaded_assets_number_of_files(self): """