Improve tests

This expands the test A LOT!
nsg · Jul 27, 2023 · 4e558c1 · 4e558c1
1 parent 4b7b18e
commit 4e558c1
Show file tree

Hide file tree

Showing 13 changed files with 232 additions and 6 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,3 +6,5 @@ __pycache__
 /docs/.cache
 /docs/.venv
 /docs/site
+/tests/secret.txt
+/tests/latest_logs
diff --git a/docs/docs/build/tests.md b/docs/docs/build/tests.md
@@ -0,0 +1,67 @@
+# Tests
+
+The rests are triggered as part of an GitHub Action in any Pull Request you open.
+
+## Run the tests locally
+
+In the `tests/` folder, run `make selenium` to start a selenium container in podman. In another terminal run `make test` to fire up the tests.
+
+## HAProxy test
+
+`test_haproxy.sh` queries the HAProxy stats endpoint, it will fail if there is any DOWN backens. I use this like below to wait for the different components to start. It usually takes around a minute or two for the state to get ready.
+
+```makefile
+wait:
+	while ! ./test_haproxy.sh ; do sleep 1; done
+```
+
+The HAProxy tests are configured like this:
+
+```
+backend be_microservices
+    {==option httpchk==}
+    {==http-check send meth GET uri /ping==}
+    server microservices 127.0.0.1:3003 maxconn 32 {==check==} inter 10s fall 2 rise 6
+```
+
+To summarize, HAProxy do a few basic backend checks to make sure that the services runs correctly. I use this information to detect if everyting is started and behaves correctly. This is exposed with the `test_haproxy.sh` script.
+
+## Selenium tests
+
+These tests are triggered when the above tests succeeds. If you have started the selenium container, you can inspect the selenium browser process at [localhost:7900](http://localhost:7900) (password: `secret`).
+
+The selenium tests are inside `tests_selenium.py`. The tests is a mix of browser tests, API calls and cli commands. At the moment the test contains these steps:
+
+1. Register a user
+2. Log in with the above user and check for JS and 404 errors
+3. Make sure that we have an empty timeline
+4. Make an API key
+5. Verify that we can execute the CLI
+6. Upload all assets in `tests/assets` to Immich via the CLI using the API key
+7. Use the API to list all assets, this test will check that:
+    * Immich contains all expected assets
+    * Our videos are detected as videos
+    * Exif extraction works on both videos and pictures
+    * Our VP9 video is transcoded to MPEG4
+    * Live/Motion photos are detected
+8. Verify that our six people are detected and processed
+
+??? Warning "Hard to match elements in `tests_selenium.py`"
+    The generated HTML is a mess, it looks like the sample below. `id` attributes and unique classes are rare.
+
+    > &lt;div class="w-[500px] max-w-[95vw] rounded-3xl border bg-immich-bg p-4 py-8 shadow-sm dark:border-immich-dark-gray dark:bg-immich-dark-gray dark:text-immich-dark-fg"&gt;&lt;div class="flex flex-col place-content-center place-items-center gap-4 px-4 text-immich-primary dark:text-immich-dark-primary"&gt;...
+
+    So I'm forced to use CSS selectors on content, or Beautiful Soup + code to process the DOM to generate unique paths to select against.
+
+    ```python
+    self.assert_element("p:contains('CLICK TO UPLOAD YOUR FIRST PHOTO')")
+    ```
+
+    ```python
+    soup = self.get_beautiful_soup()
+    api_keys_div = soup.find(string="API Keys").parent.parent.parent
+    api_keys_button = api_keys_div.find("button")
+    self.click(css_selector_path(api_keys_button))
+    ```
+
+I hope to add more test over time, like more filetypes. If you like to contribute a file with an appropriate license, open an issue.
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
@@ -17,6 +17,7 @@ nav:
   - Contribute:
     - Upgrade: build/upgrade.md
     - Patches: build/patches.md
+    - Tests: build/tests.md
 
 theme:
   name: material

diff --git a/tests/assets/ai-apple.tiff b/tests/assets/ai-apple.tiff
diff --git a/tests/assets/ai-people1.png b/tests/assets/ai-people1.png
diff --git a/tests/assets/ai-people2.png b/tests/assets/ai-people2.png
diff --git a/tests/assets/ai-people3.png b/tests/assets/ai-people3.png
diff --git a/tests/assets/field.jpg b/tests/assets/field.jpg
diff --git a/tests/assets/grass.MP.jpg b/tests/assets/grass.MP.jpg
diff --git a/tests/assets/ohm.gif b/tests/assets/ohm.gif
diff --git a/tests/assets/ship-vp9.webm b/tests/assets/ship-vp9.webm
diff --git a/tests/assets/ship.mp4 b/tests/assets/ship.mp4
diff --git a/tests/tests_selenium.py b/tests/tests_selenium.py
@@ -1,8 +1,11 @@
 #!/usr/bin/env python3
 
 import socket
-import time
+import os
 import subprocess
+import shutil
+import requests
+
 from seleniumbase import BaseCase
 BaseCase.main(__name__, __file__)
 
@@ -11,11 +14,33 @@ def get_ip_address():
     s.connect(("8.8.8.8", 80))
     return s.getsockname()[0]
 
+def is_dirty_state():
+    return os.getenv("DIRTY_STATE")
+
+
+def css_selector_path(element):
+    """ Returns a CSS selector that will uniquely select the given element. """
+    path = []
+    while element.parent:
+        siblings = element.parent.find_all(element.name, recursive=False)
+        if len(siblings) > 1:
+            index = siblings.index(element)
+            path.insert(0, f"{element.name}:nth-child({index+1})")
+        else:
+            path.insert(0, element.name)
+        element = element.parent
+
+    return " > ".join(path)
 
 class TestImmichWeb(BaseCase):
 
-    def immich(self, path=""):
+    def immich(self, path="", login=True):
         self.open(f"http://{get_ip_address()}/{path}")
+        if login:
+            self.login()
+            self.sleep(2)
+            if self.is_element_present("button:contains('Acknowledge')"):
+                self.click("button:contains('Acknowledge')")
 
     def register(self):
         # Welcome page, click button
@@ -33,10 +58,141 @@ def login(self):
         self.type("input[id='email']", "foo@example.com")
         self.type("input[id='password']", "secret")
         self.click("button")
+        self.wait_for_element("p:contains('Photos')")
 
     def test_001_register_and_login(self):
-        self.immich()
-        self.register()
+        if not is_dirty_state():
+            self.immich(login=False)
+            self.register()
+            self.login()
+            self.assert_title("Photos - Immich")
+
+    def test_002_no_errors(self):
+        self.immich(login=False)
+        self.assert_no_js_errors()
+        self.assert_no_404_errors()
         self.login()
-        time.sleep(20)
-        self.assert_title("Photos - Immich")
+        self.assert_no_js_errors()
+        self.assert_no_404_errors()
+
+    def test_003_empty_timeline(self):
+        if not is_dirty_state():
+            self.immich()
+            self.assert_element("p:contains('CLICK TO UPLOAD YOUR FIRST PHOTO')")
+
+    def test_004_generate_api_keys(self):
+        self.immich(login=True)
+        self.immich(login=False, path="user-settings")
+        self.wait_for_element("h2")
+        soup = self.get_beautiful_soup()
+        api_keys_div = soup.find(string="API Keys").parent.parent.parent
+        api_keys_button = api_keys_div.find("button")
+        self.click(css_selector_path(api_keys_button))
+        self.click("button:contains('New API Key')")
+        self.type("input[id='name']", "test\n")
+        secret = self.get_text("textarea[id='secret']")
+        with open("secret.txt", "w") as f:
+            f.write(secret)
+
+    def test_10_verify_cli(self):
+        p = subprocess.run(["immich-distribution.cli", "-h"])
+        self.assertEqual(p.returncode, 0)
+
+    def test_005_upload_assets_with_cli(self):
+        with open("secret.txt", "r") as f:
+            secret = f.read()
+
+        snap_readable_path = os.path.join(
+            os.environ["HOME"],
+            "snap/immich-distribution/current/tests"
+        )
+
+        if not os.path.exists(snap_readable_path):
+            os.mkdir(snap_readable_path)
+
+        for upload in os.listdir("assets"):
+            shutil.copy(f"assets/{upload}", snap_readable_path)
+
+        subprocess.run(
+            [
+                "immich-distribution.cli",
+                "upload",
+                "--key", secret,
+                "--yes",
+                snap_readable_path
+            ]
+        )
+
+        # Give the system time to process the new assets
+        self.sleep(60)
+
+    def test_100_verify_uploaded_assets(self):
+        """
+        Query the API to get a list of assets, this mainly tests that:
+          * The assets are there (upload works)
+          * The exif extraction works
+          * The live photo detection works
+          * FFmpeg transcoding works
+        """
+
+        with open("secret.txt", "r") as f:
+            secret = f.read()
+
+        headers = { "X-API-KEY": secret }
+
+        r = requests.get(f"http://{get_ip_address()}/api/asset", headers=headers)
+        assets = r.json()
+        self.assertEqual(len(assets), 11)
+
+        for asset in assets:
+            match asset['originalFileName']:
+                case "ship":
+                    self.assertEqual(asset['type'], "VIDEO")
+                    self.assertEqual(asset['exifInfo']['country'], "Sweden")
+                case "ship-vp9":
+                    self.assertEqual(asset['type'], "VIDEO")
+
+                    r = requests.get(f"http://{get_ip_address()}/api/asset/file/{asset['id']}?isThumb=false&isWeb=true", headers=headers)
+                    self.assertEqual(r.status_code, 200)
+                    self.assertEqual(r.headers['content-type'], "video/mp4")
+                case "ohm":
+                    self.assertEqual(asset['type'], "IMAGE")
+                    self.assertEqual(asset['exifInfo']['exifImageWidth'], 640)
+                case "grass.MP":
+                    self.assertEqual(asset['type'], "IMAGE")
+                    self.assertEqual(asset['exifInfo']['model'], "Pixel 4")
+                    self.assertEqual(asset['exifInfo']['dateTimeOriginal'], "2023-07-08T12:13:53.000Z")
+                    self.assertEqual(asset['exifInfo']['city'], "Mora")
+                    self.assertNotEqual(asset['livePhotoVideoId'], None)
+                case "plane":
+                    pass
+                case "field":
+                    pass
+                case "memory":
+                    pass
+                case "ai-people1":
+                    pass
+                case "ai-people2":
+                    pass
+                case "ai-people3":
+                    pass
+                case "ai-apple":
+                    pass
+                case _:
+                    raise Exception(asset)
+
+    def test_100_verify_people_detected(self):
+        """
+        Query the API to get a list of assets, this mainly tests that:
+          * The ML model works and detects people
+          * Typesese works (used to generate embeddings)
+        """
+        with open("secret.txt", "r") as f:
+            secret = f.read()
+
+        headers = { "X-API-KEY": secret }
+
+        # query the API to get a list of people
+        r = requests.get(f"http://{get_ip_address()}/api/person", headers=headers)
+        people = r.json()
+        self.assertEqual(people['total'], 6)