Merge pull request #102 from nsg/bump/v1.77.0

Bump v1.77.0

.gitignore

@@ -10,3 +10,5 @@ __pycache__
 /tests/latest_logs
 /tests/report.html
 /tests/assets/style.css
+/tests/assets/tests_selenium.py__*
+/tests/external-test-files

Makefile

@@ -11,6 +11,10 @@ build:
 install:
 	sudo snap install --dangerous ${SNAP_FILE}
 
+.PHONY: remove
+remove:
+	sudo snap remove --purge immich-distribution
+
 beta: build
 	cat ${SNAP_FILE} | ssh d -- lxc file push - immich-beta/root/${SNAP_FILE}
 	ssh d lxc exec immich-beta -- snap install --dangerous /root/${SNAP_FILE}
@@ -21,6 +25,14 @@ beta2store:
 shell:
 	multipass shell snapcraft-immich-distribution
 
+.PHONY: tests
+tests:
+	make -C tests test
+
+.PHONY: selenium
+selenium:
+	make -C tests/ selenium
+
 .PHONY: docs
 docs:
 	cd docs && poetry run mkdocs serve

VERSION

@@ -1 +1 @@
-v1.76.1
+v1.77.0

parts/machine-learning/Makefile

@@ -1,11 +1,52 @@
+IMMICH_VERSION := v1.77.0
+
+# Use our own Python, over the older one in core20
+export PATH := ${SNAPCRAFT_PART_BUILD}/../../dependencies/install/usr/local/bin:$(PATH)
+
+POETRY_ENV = ${HOME}/poetry
+POETRY = ${POETRY_ENV}/bin/poetry
+
 .PHONY: build
-build:
-	./build.sh v1.76.1
+build: immich ${POETRY_ENV}
+	${POETRY} install \
+		-C immich/machine-learning \
+		--sync \
+		--no-interaction \
+		--no-ansi \
+		--no-root \
+		--only main
+
+	${POETRY} run \
+		-C immich/machine-learning \
+		pip install \
+			--no-deps \
+			-r immich/machine-learning/requirements.txt
+
+immich:
+	git clone --branch ${IMMICH_VERSION} https://github.com/immich-app/immich.git
+
+${POETRY_ENV}:
+	python3 -m venv ${POETRY_ENV}
+	${POETRY_ENV}/bin/pip install -U pip setuptools
+	${POETRY_ENV}/bin/pip install poetry
 
 .PHONY: install
 install:
-	mkdir -p ${SNAPCRAFT_PART_INSTALL}/usr/src/ml
-	mkdir -p ${SNAPCRAFT_PART_INSTALL}/opt/python-libs/ml
-	cp -r immich/machine-learning/app ${SNAPCRAFT_PART_INSTALL}/usr/src/ml
-	cp -r /opt/venv/ml/lib/python3.11/site-packages \
-		${SNAPCRAFT_PART_INSTALL}/opt/python-libs/ml/
+	mkdir -p ${SNAPCRAFT_PART_INSTALL}/app/machine-learning
+	mkdir -p ${SNAPCRAFT_PART_INSTALL}/bin
+
+	# Deploy application
+	cp immich/machine-learning/log_conf.json ${SNAPCRAFT_PART_INSTALL}/app/machine-learning/
+	cp -r immich/machine-learning/app ${SNAPCRAFT_PART_INSTALL}/app/machine-learning/app
+
+	# Deploy Python packages (installed with Poetry & pip in the build step)
+	cp -r $(shell ${POETRY} -C immich/machine-learning env info -p)/lib/python3.11/site-packages \
+		${SNAPCRAFT_PART_INSTALL}/app/machine-learning/python-packages
+
+	# Deploy gunicorn (installed with Poetry) and remove shebang
+	cp $(shell ${POETRY} -C immich/machine-learning env info -p)/bin/gunicorn \
+		${SNAPCRAFT_PART_INSTALL}/bin/gunicorn
+	sed -i '1d' ${SNAPCRAFT_PART_INSTALL}/bin/gunicorn
+
+	# Deploy immich-machine-learning script
+	cp immich-machine-learning ${SNAPCRAFT_PART_INSTALL}/bin/immich-machine-learning

parts/machine-learning/immich-machine-learning

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+. $SNAP/bin/load-env
+
+export PYTHONPATH="$SNAP/app/machine-learning/python-packages"
+export LD_PRELOAD="$SNAP/usr/local/lib/libmimalloc.so.2"
+
+echo "Launching machine-learning service using python version: $(python3 --version)"
+echo "PYTHONPATH: $PYTHONPATH"
+echo "LD_PRELOAD: $LD_PRELOAD"
+
+cd $SNAP/app/machine-learning
+python3 $SNAP/bin/gunicorn app.main:app \
+ 	-k uvicorn.workers.UvicornWorker \
+ 	-w $MACHINE_LEARNING_WORKERS \
+ 	-b $MACHINE_LEARNING_HOST:$MACHINE_LEARNING_PORT \
+ 	--log-config-json log_conf.json

patches/Makefile

@@ -1,4 +1,4 @@
-VERSION=v1.76.1
+VERSION=v1.77.0
 SOURCE_FILE_PATH=https://raw.githubusercontent.com/immich-app/immich/${VERSION}/web/src/lib/components/shared-components/version-announcement-box.svelte
 
 download-patch:

snap/snapcraft.yaml

@@ -182,7 +182,7 @@ parts:
     plugin: npm
     npm-node-version: "18.16.0"
     source: https://github.com/immich-app/immich.git
-    source-tag: v1.76.1
+    source-tag: v1.77.0
     source-subdir: server
     override-build: |
       snapcraftctl set-version "$(git describe --tags)-dist1"
@@ -203,7 +203,7 @@ parts:
     plugin: npm
     npm-node-version: "18.16.0"
     source: https://github.com/immich-app/immich.git
-    source-tag: v1.76.1
+    source-tag: v1.77.0
     source-subdir: web
     override-build: |
       patch -p0 -i $SNAPCRAFT_PART_SRC/../../patches/src/001-version-announcement-box.patch -d $SNAPCRAFT_PART_BUILD
@@ -255,6 +255,7 @@ parts:
       - immich-distribution-libvips/latest/stable
       - immich-distribution-node/latest/stable
       - immich-distribution-python/latest/stable
+      - immich-distribution-mimalloc/latest/stable
 
   scripts:
     source: src

src/bin/load-env

@@ -8,6 +8,7 @@ snapctl() {
     fi
 }
 
+export PATH="$SNAP/usr/local/bin:$PATH"
 export LD_LIBRARY_PATH="$SNAP/usr/local/pgsql/lib:$SNAP/usr/lib/x86_64-linux-gnu/pulseaudio:$SNAP/usr/lib/x86_64-linux-gnu/libfswatch:$SNAP/usr/local/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH"
 
 export PGDATA="$SNAP_COMMON/pgsql/data"
@@ -32,6 +33,10 @@ export IMMICH_SERVER_ADDRESS="$IMMICH_SERVER_URL" # CLI
 export IMMICH_MACHINE_LEARNING_URL="http://127.0.0.1:3003"
 export MACHINE_LEARNING_LOG_LEVEL="warning"
 export MACHINE_LEARNING_IP="127.0.0.1"
+export MACHINE_LEARNING_CACHE_FOLDER="$SNAP_COMMON/cache"
+export MACHINE_LEARNING_HOST="0.0.0.0"
+export MACHINE_LEARNING_PORT=3003
+export MACHINE_LEARNING_WORKERS=1
 
 export TYPESENSE_API_KEY="$(snapctl get typesense-key)"
 export TYPESENSE_DATA_DIR="$SNAP_COMMON/typesense"

src/etc/haproxy.cfg

@@ -54,12 +54,12 @@ backend be_typesense
 backend be_microservices
     option httpchk
     http-check send meth GET uri /ping
-    server microservices 127.0.0.1:3003 maxconn 32 check inter 10s fall 2 rise 6
+    server microservices 127.0.0.1:3003 maxconn 7 check inter 10s fall 2 rise 6
 
 backend be_ml
     option httpchk
     http-check send meth GET uri /ping
-    server ml 127.0.0.1:3003 maxconn 32 check inter 10s fall 2 rise 6
+    server ml 127.0.0.1:3003 maxconn 7 check inter 10s fall 2 rise 6
 
 backend be_postgres
     mode tcp

tests/Makefile

@@ -1,22 +1,42 @@
-
-test: .env/bin/seleniumbase wait
+test: .env/bin/seleniumbase prepare-test-file wait check-selenium
 	.env/bin/pytest tests_selenium.py --server=127.0.0.1 --port=4444 --headed --html=report.html
 
-trace: .env/bin/seleniumbase wait
+trace: .env/bin/seleniumbase prepare-test-file wait
 	.env/bin/pytest tests_selenium.py --server=127.0.0.1 --port=4444 --headed --html=report.html --trace
 
-ci: wait
+ci: prepare-test-file wait
 	pytest tests_selenium.py --server=127.0.0.1 --port=4444 --html=report.html
 
+prepare-test-file: external-test-files update-snap-symlinks-for-user
+	mkdir -p ${HOME}/snap/immich-distribution/current/tests
+	mkdir -p ${HOME}/snap/immich-distribution/current/tests_external
+	for asset in assets/*; do \
+		echo $$asset | grep -q py__ \
+			|| cp $$asset ${HOME}/snap/immich-distribution/current/tests/; \
+	done
+	cp -r external-test-files ${HOME}/snap/immich-distribution/current/tests_external
+
+external-test-files: update-snap-symlinks-for-user
+	mkdir -p external-test-files
+
+	cd external-test-files && for file in $(shell cat external-test-files.list); do \
+		wget -c $$file; \
+	done
+
+update-snap-symlinks-for-user:
+	immich-distribution.cli -h > /dev/null 2>&1
+
 wait:
 	./test_haproxy.sh
 
+check-selenium:
+	nc -v -w1 127.0.0.1 4444 || (echo "Selenium is not running"; exit 1)
+
 selenium:
 	podman run \
 		-p 4444:4444 \
 		-p 7900:7900 \
 		--shm-size="2g" \
-		-v ${PWD}/assets:/assets \
 		docker.io/selenium/standalone-chrome:latest
 
 .env:

tests/tests_selenium.py

@@ -5,6 +5,7 @@
 import subprocess
 import shutil
 import requests
+import time
 
 from seleniumbase import BaseCase
 BaseCase.main(__name__, __file__)
@@ -37,6 +38,26 @@ def get_assets(filter=[]):
 
     return resp
 
+def get_all_jobs():
+    r = requests.get(f"http://{get_ip_address()}/api/jobs", headers=get_headers())
+    return r.json()
+
+def trigger_job(job_name):
+    r = requests.put(
+        f"http://{get_ip_address()}/api/jobs/{job_name}",
+        headers=get_headers(),
+        json={"command": "start"}
+    )
+    return r.json()
+
+def wait_for_empty_job_queue():
+    for job_name, job_data in get_all_jobs().items():
+        status = job_data['queueStatus']['isActive']
+        if status == True:
+            print(f"Queue {job_name} is running")
+            time.sleep(1)
+            return wait_for_empty_job_queue()
+
 def css_selector_path(element):
     """ Returns a CSS selector that will uniquely select the given element. """
     path = []
@@ -141,27 +162,37 @@ def test_005_upload_assets_with_cli(self):
 
         snap_readable_path = os.path.join(
             os.environ["HOME"],
-            "snap/immich-distribution/current/tests"
+            "snap/immich-distribution/current/"
         )
 
-        if not os.path.exists(snap_readable_path):
-            os.makedirs(snap_readable_path)
-
-        for upload in os.listdir("assets"):
-            shutil.copy(f"assets/{upload}", snap_readable_path)
+        subprocess.run(
+            [
+                "immich-distribution.cli",
+                "upload",
+                "--key", secret,
+                "--yes",
+                f"{snap_readable_path}/tests"
+            ]
+        )
 
         subprocess.run(
             [
                 "immich-distribution.cli",
                 "upload",
                 "--key", secret,
                 "--yes",
-                snap_readable_path
+                f"{snap_readable_path}/tests_external"
             ]
         )
 
-        # Give the system time to process the new assets
-        self.sleep(60)
+        # ML models are downloaded in the background when we upload assets
+        # Wait for them to complete, and the queue to be empty before continuing
+        wait_for_empty_job_queue()
+
+        # Re-run the recognition job. I'm not sure if this is an Immich bug or
+        # just a quirk of the test environment. Anyway let's just run it again.
+        trigger_job("recognizeFaces")
+        wait_for_empty_job_queue()
 
     def test_100_verify_uploaded_assets_number_of_files(self):
         """

update.sh

@@ -56,8 +56,10 @@ CHECK_FILES="
     nginx
     machine-learning/README.md
     machine-learning/Dockerfile
+    machine-learning/start.sh
     docker/example.env
     docker/docker-compose.yml
+    docs/docs/install/environment-variables.md
 "
 
 for F in $CHECK_FILES; do