diff --git a/.gitignore b/.gitignore index 9873055..37ebc05 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ __pycache__ /tests/latest_logs /tests/report.html /tests/assets/style.css +/tests/assets/tests_selenium.py__* +/tests/external-test-files diff --git a/Makefile b/Makefile index 9d5be9b..ba94a2e 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,10 @@ build: install: sudo snap install --dangerous ${SNAP_FILE} +.PHONY: remove +remove: + sudo snap remove --purge immich-distribution + beta: build cat ${SNAP_FILE} | ssh d -- lxc file push - immich-beta/root/${SNAP_FILE} ssh d lxc exec immich-beta -- snap install --dangerous /root/${SNAP_FILE} @@ -21,6 +25,14 @@ beta2store: shell: multipass shell snapcraft-immich-distribution +.PHONY: tests +tests: + make -C tests test + +.PHONY: selenium +selenium: + make -C tests/ selenium + .PHONY: docs docs: cd docs && poetry run mkdocs serve diff --git a/VERSION b/VERSION index fb868ec..1ed23ab 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.76.1 +v1.77.0 diff --git a/parts/machine-learning/Makefile b/parts/machine-learning/Makefile index 9dc8ecd..d1334d5 100644 --- a/parts/machine-learning/Makefile +++ b/parts/machine-learning/Makefile @@ -1,11 +1,52 @@ +IMMICH_VERSION := v1.77.0 + +# Use our own Python, over the older one in core20 +export PATH := ${SNAPCRAFT_PART_BUILD}/../../dependencies/install/usr/local/bin:$(PATH) + +POETRY_ENV = ${HOME}/poetry +POETRY = ${POETRY_ENV}/bin/poetry + .PHONY: build -build: - ./build.sh v1.76.1 +build: immich ${POETRY_ENV} + ${POETRY} install \ + -C immich/machine-learning \ + --sync \ + --no-interaction \ + --no-ansi \ + --no-root \ + --only main + + ${POETRY} run \ + -C immich/machine-learning \ + pip install \ + --no-deps \ + -r immich/machine-learning/requirements.txt + +immich: + git clone --branch ${IMMICH_VERSION} https://github.com/immich-app/immich.git + +${POETRY_ENV}: + python3 -m venv ${POETRY_ENV} + ${POETRY_ENV}/bin/pip install -U pip setuptools + ${POETRY_ENV}/bin/pip install poetry .PHONY: install install: - mkdir -p ${SNAPCRAFT_PART_INSTALL}/usr/src/ml - mkdir -p ${SNAPCRAFT_PART_INSTALL}/opt/python-libs/ml - cp -r immich/machine-learning/app ${SNAPCRAFT_PART_INSTALL}/usr/src/ml - cp -r /opt/venv/ml/lib/python3.11/site-packages \ - ${SNAPCRAFT_PART_INSTALL}/opt/python-libs/ml/ + mkdir -p ${SNAPCRAFT_PART_INSTALL}/app/machine-learning + mkdir -p ${SNAPCRAFT_PART_INSTALL}/bin + + # Deploy application + cp immich/machine-learning/log_conf.json ${SNAPCRAFT_PART_INSTALL}/app/machine-learning/ + cp -r immich/machine-learning/app ${SNAPCRAFT_PART_INSTALL}/app/machine-learning/app + + # Deploy Python packages (installed with Poetry & pip in the build step) + cp -r $(shell ${POETRY} -C immich/machine-learning env info -p)/lib/python3.11/site-packages \ + ${SNAPCRAFT_PART_INSTALL}/app/machine-learning/python-packages + + # Deploy gunicorn (installed with Poetry) and remove shebang + cp $(shell ${POETRY} -C immich/machine-learning env info -p)/bin/gunicorn \ + ${SNAPCRAFT_PART_INSTALL}/bin/gunicorn + sed -i '1d' ${SNAPCRAFT_PART_INSTALL}/bin/gunicorn + + # Deploy immich-machine-learning script + cp immich-machine-learning ${SNAPCRAFT_PART_INSTALL}/bin/immich-machine-learning diff --git a/parts/machine-learning/build.sh b/parts/machine-learning/build.sh deleted file mode 100755 index 6e9dbf0..0000000 --- a/parts/machine-learning/build.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -VERSION="$1" -PYTHON_PART_INSTALL_BIN="${PWD}/../../dependencies/install/usr/local/bin" - -mkdir -p /opt/venv/ - -python() { - $PYTHON_PART_INSTALL_BIN/python3 $@ -} - -pip() { - python $PYTHON_PART_INSTALL_BIN/pip3 $@ -} - -[ -d immich ] || git clone --branch $VERSION https://github.com/immich-app/immich.git -[ -d "/opt/venv/ml" ] || python -m venv "/opt/venv/ml" -[ -d "/opt/venv/poetry" ] || python -m venv "/opt/venv/poetry" - -/opt/venv/poetry/bin/python --version -/opt/venv/poetry/bin/pip install poetry - -# Configure Poetry and install dependencies -cd immich/machine-learning - -export VIRTUAL_ENV="/opt/venv/ml" -/opt/venv/poetry/bin/poetry config installer.max-workers 10 -/opt/venv/poetry/bin/poetry config virtualenvs.create false -/opt/venv/poetry/bin/poetry install --sync --no-interaction --no-ansi --no-root --only main -/opt/venv/poetry/bin/poetry run pip install --no-deps -r requirements.txt -unset VIRTUAL_ENV diff --git a/parts/machine-learning/immich-machine-learning b/parts/machine-learning/immich-machine-learning new file mode 100755 index 0000000..ad4ab66 --- /dev/null +++ b/parts/machine-learning/immich-machine-learning @@ -0,0 +1,17 @@ +#!/bin/bash + +. $SNAP/bin/load-env + +export PYTHONPATH="$SNAP/app/machine-learning/python-packages" +export LD_PRELOAD="$SNAP/usr/local/lib/libmimalloc.so.2" + +echo "Launching machine-learning service using python version: $(python3 --version)" +echo "PYTHONPATH: $PYTHONPATH" +echo "LD_PRELOAD: $LD_PRELOAD" + +cd $SNAP/app/machine-learning +python3 $SNAP/bin/gunicorn app.main:app \ + -k uvicorn.workers.UvicornWorker \ + -w $MACHINE_LEARNING_WORKERS \ + -b $MACHINE_LEARNING_HOST:$MACHINE_LEARNING_PORT \ + --log-config-json log_conf.json diff --git a/patches/Makefile b/patches/Makefile index 577b578..76d7c62 100644 --- a/patches/Makefile +++ b/patches/Makefile @@ -1,4 +1,4 @@ -VERSION=v1.76.1 +VERSION=v1.77.0 SOURCE_FILE_PATH=https://raw.githubusercontent.com/immich-app/immich/${VERSION}/web/src/lib/components/shared-components/version-announcement-box.svelte download-patch: diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index c6d3076..db0f646 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -182,7 +182,7 @@ parts: plugin: npm npm-node-version: "18.16.0" source: https://github.com/immich-app/immich.git - source-tag: v1.76.1 + source-tag: v1.77.0 source-subdir: server override-build: | snapcraftctl set-version "$(git describe --tags)-dist1" @@ -203,7 +203,7 @@ parts: plugin: npm npm-node-version: "18.16.0" source: https://github.com/immich-app/immich.git - source-tag: v1.76.1 + source-tag: v1.77.0 source-subdir: web override-build: | patch -p0 -i $SNAPCRAFT_PART_SRC/../../patches/src/001-version-announcement-box.patch -d $SNAPCRAFT_PART_BUILD @@ -255,6 +255,7 @@ parts: - immich-distribution-libvips/latest/stable - immich-distribution-node/latest/stable - immich-distribution-python/latest/stable + - immich-distribution-mimalloc/latest/stable scripts: source: src diff --git a/src/bin/immich-machine-learning b/src/bin/immich-machine-learning deleted file mode 100755 index 1b0af3c..0000000 --- a/src/bin/immich-machine-learning +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -. $SNAP/bin/load-env - -export PYTHONDONTWRITEBYTECODE=1 -export PYTHONUNBUFFERED=1 -export PYTHONPATH="$SNAP/opt/python-libs/ml/site-packages/:$SNAP/usr/src/ml" -export MACHINE_LEARNING_CACHE_FOLDER="$SNAP_COMMON/cache" - -$SNAP/usr/local/bin/python3 -m app.main | grep -v "GET /ping" diff --git a/src/bin/load-env b/src/bin/load-env index 33e2e75..4a39964 100755 --- a/src/bin/load-env +++ b/src/bin/load-env @@ -8,6 +8,7 @@ snapctl() { fi } +export PATH="$SNAP/usr/local/bin:$PATH" export LD_LIBRARY_PATH="$SNAP/usr/local/pgsql/lib:$SNAP/usr/lib/x86_64-linux-gnu/pulseaudio:$SNAP/usr/lib/x86_64-linux-gnu/libfswatch:$SNAP/usr/local/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH" export PGDATA="$SNAP_COMMON/pgsql/data" @@ -32,6 +33,10 @@ export IMMICH_SERVER_ADDRESS="$IMMICH_SERVER_URL" # CLI export IMMICH_MACHINE_LEARNING_URL="http://127.0.0.1:3003" export MACHINE_LEARNING_LOG_LEVEL="warning" export MACHINE_LEARNING_IP="127.0.0.1" +export MACHINE_LEARNING_CACHE_FOLDER="$SNAP_COMMON/cache" +export MACHINE_LEARNING_HOST="0.0.0.0" +export MACHINE_LEARNING_PORT=3003 +export MACHINE_LEARNING_WORKERS=1 export TYPESENSE_API_KEY="$(snapctl get typesense-key)" export TYPESENSE_DATA_DIR="$SNAP_COMMON/typesense" diff --git a/src/etc/haproxy.cfg b/src/etc/haproxy.cfg index 2027e68..f4e3976 100644 --- a/src/etc/haproxy.cfg +++ b/src/etc/haproxy.cfg @@ -54,12 +54,12 @@ backend be_typesense backend be_microservices option httpchk http-check send meth GET uri /ping - server microservices 127.0.0.1:3003 maxconn 32 check inter 10s fall 2 rise 6 + server microservices 127.0.0.1:3003 maxconn 7 check inter 10s fall 2 rise 6 backend be_ml option httpchk http-check send meth GET uri /ping - server ml 127.0.0.1:3003 maxconn 32 check inter 10s fall 2 rise 6 + server ml 127.0.0.1:3003 maxconn 7 check inter 10s fall 2 rise 6 backend be_postgres mode tcp diff --git a/tests/Makefile b/tests/Makefile index 343f541..8c36268 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,22 +1,42 @@ - -test: .env/bin/seleniumbase wait +test: .env/bin/seleniumbase prepare-test-file wait check-selenium .env/bin/pytest tests_selenium.py --server=127.0.0.1 --port=4444 --headed --html=report.html -trace: .env/bin/seleniumbase wait +trace: .env/bin/seleniumbase prepare-test-file wait .env/bin/pytest tests_selenium.py --server=127.0.0.1 --port=4444 --headed --html=report.html --trace -ci: wait +ci: prepare-test-file wait pytest tests_selenium.py --server=127.0.0.1 --port=4444 --html=report.html +prepare-test-file: external-test-files update-snap-symlinks-for-user + mkdir -p ${HOME}/snap/immich-distribution/current/tests + mkdir -p ${HOME}/snap/immich-distribution/current/tests_external + for asset in assets/*; do \ + echo $$asset | grep -q py__ \ + || cp $$asset ${HOME}/snap/immich-distribution/current/tests/; \ + done + cp -r external-test-files ${HOME}/snap/immich-distribution/current/tests_external + +external-test-files: update-snap-symlinks-for-user + mkdir -p external-test-files + + cd external-test-files && for file in $(shell cat external-test-files.list); do \ + wget -c $$file; \ + done + +update-snap-symlinks-for-user: + immich-distribution.cli -h > /dev/null 2>&1 + wait: ./test_haproxy.sh +check-selenium: + nc -v -w1 127.0.0.1 4444 || (echo "Selenium is not running"; exit 1) + selenium: podman run \ -p 4444:4444 \ -p 7900:7900 \ --shm-size="2g" \ - -v ${PWD}/assets:/assets \ docker.io/selenium/standalone-chrome:latest .env: diff --git a/tests/external-test-files.list b/tests/external-test-files.list new file mode 100644 index 0000000..e69de29 diff --git a/tests/tests_selenium.py b/tests/tests_selenium.py index b816aca..a8dcf61 100755 --- a/tests/tests_selenium.py +++ b/tests/tests_selenium.py @@ -5,6 +5,7 @@ import subprocess import shutil import requests +import time from seleniumbase import BaseCase BaseCase.main(__name__, __file__) @@ -37,6 +38,26 @@ def get_assets(filter=[]): return resp +def get_all_jobs(): + r = requests.get(f"http://{get_ip_address()}/api/jobs", headers=get_headers()) + return r.json() + +def trigger_job(job_name): + r = requests.put( + f"http://{get_ip_address()}/api/jobs/{job_name}", + headers=get_headers(), + json={"command": "start"} + ) + return r.json() + +def wait_for_empty_job_queue(): + for job_name, job_data in get_all_jobs().items(): + status = job_data['queueStatus']['isActive'] + if status == True: + print(f"Queue {job_name} is running") + time.sleep(1) + return wait_for_empty_job_queue() + def css_selector_path(element): """ Returns a CSS selector that will uniquely select the given element. """ path = [] @@ -141,14 +162,18 @@ def test_005_upload_assets_with_cli(self): snap_readable_path = os.path.join( os.environ["HOME"], - "snap/immich-distribution/current/tests" + "snap/immich-distribution/current/" ) - if not os.path.exists(snap_readable_path): - os.makedirs(snap_readable_path) - - for upload in os.listdir("assets"): - shutil.copy(f"assets/{upload}", snap_readable_path) + subprocess.run( + [ + "immich-distribution.cli", + "upload", + "--key", secret, + "--yes", + f"{snap_readable_path}/tests" + ] + ) subprocess.run( [ @@ -156,12 +181,18 @@ def test_005_upload_assets_with_cli(self): "upload", "--key", secret, "--yes", - snap_readable_path + f"{snap_readable_path}/tests_external" ] ) - # Give the system time to process the new assets - self.sleep(60) + # ML models are downloaded in the background when we upload assets + # Wait for them to complete, and the queue to be empty before continuing + wait_for_empty_job_queue() + + # Re-run the recognition job. I'm not sure if this is an Immich bug or + # just a quirk of the test environment. Anyway let's just run it again. + trigger_job("recognizeFaces") + wait_for_empty_job_queue() def test_100_verify_uploaded_assets_number_of_files(self): """ diff --git a/update.sh b/update.sh index 6d07ea0..7752337 100755 --- a/update.sh +++ b/update.sh @@ -56,8 +56,10 @@ CHECK_FILES=" nginx machine-learning/README.md machine-learning/Dockerfile + machine-learning/start.sh docker/example.env docker/docker-compose.yml + docs/docs/install/environment-variables.md " for F in $CHECK_FILES; do