Reenable notebook tests (#178)

* More sleep
* Remove pylint 'noqa's
* Enlarge window
* Don't install Firefox manually

.github/workflows/ci.yml

@@ -3,25 +3,66 @@
 
 name: continuous-integration
 
-on: [push, pull_request]
+on:
+    push:
+        branches:
+            - master
+    pull_request:
+
+env:
+    FORCE_COLOR: '1'
+
+# https://docs.github.com/en/actions/using-jobs/using-concurrency
+concurrency:
+    # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
 
 jobs:
 
-    pre-commit:
+    test-notebooks:
+
+        strategy:
+            matrix:
+                browser: [Chrome, Firefox]
+            fail-fast: false
 
         runs-on: ubuntu-latest
+        timeout-minutes: 30
 
         steps:
-            - uses: actions/checkout@v4
 
-            - name: Setup Python
+            - name: Check out app
+              uses: actions/checkout@v4
+
+            - name: Set up Python
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.11
+                  python-version: '3.11'
 
-            - name: Install dependencies
-              run: |
-                  pip install .[dev]
+            - name: Setup uv
+              uses: astral-sh/setup-uv@v3
+              with:
+                  version: 0.4.20
 
-            - name: Run pre-commit
-              run: pre-commit run --all-files || ( git status --short ; git diff ; exit 1 )
+            - name: Install package test dependencies
+              # Notebook tests happen in the container, here we only need to install
+              # the pytest-docker dependency. Unfortunately, uv/pip does not allow to
+              # only install [dev] dependencies so we end up installing all the rest as well.
+              run: uv pip install --system .[dev]
+
+            - name: Set jupyter token env
+              run: echo "JUPYTER_TOKEN=$(openssl rand -hex 32)" >> $GITHUB_ENV
+
+            - name: Run pytest
+              run: pytest -v --driver ${{ matrix.browser }} tests_notebooks
+              env:
+                  TAG: edge
+
+            - name: Upload screenshots as artifacts
+              if: always()
+              uses: actions/upload-artifact@v4
+              with:
+                  name: Screenshots-${{ matrix.browser }}
+                  path: screenshots/
+                  if-no-files-found: error

home/utils.py

@@ -32,7 +32,7 @@ def load_start_py(name):
             )
         except TypeError:
             return mod.get_start_widget(appbase=appbase, jupbase=jupbase)
-    except Exception:  # pylint: disable=broad-except
+    except Exception:
         return ipw.HTML(f"<pre>{sys.exc_info()}</pre>")
 
 
@@ -51,7 +51,7 @@ def load_start_md(name):
         html = html.replace("<h3", "<h4")
         return ipw.HTML(html)
 
-    except Exception as exc:  # pylint: disable=broad-except
+    except Exception as exc:
         return ipw.HTML(f"Could not load start.md: {exc!s}")
 
 

home/widgets.py

@@ -142,7 +142,7 @@ def __init__(self, spinner_style=None):
         super().__init__()
 
     @traitlets.default("enabled")
-    def _default_enabled(self):  # pylint: disable=no-self-use
+    def _default_enabled(self):
         return False
 
     @traitlets.observe("enabled")

pyproject.toml

@@ -28,4 +28,4 @@ select = [
 ]
 
 [tool.ruff.lint.per-file-ignores]
-"tests/*" = ["ARG001"]
+"tests_notebooks/*" = ["ARG001"]

setup.cfg

@@ -37,6 +37,10 @@ python_requires = >=3.9
 dev =
     bumpver==2022.1118
     pre-commit==3.6.0
+    pytest~=8.3.0
+    pytest-docker~=3.1.0
+    pytest-selenium~=4.1.0
+    selenium~=4.23.0
 
 [flake8]
 ignore =

single_app.ipynb

@@ -41,9 +41,8 @@
     "url = urlparse.urlsplit(jupyter_notebook_url)  # noqa: F821\n",
     "try:\n",
     "    name = urlparse.parse_qs(url.query)[\"app\"][0]\n",
-    "except KeyError:\n",
-    "    raise Exception(\"No app specified\")  # noqa: TRY002\n",
-    "    exit()"
+    "except KeyError as e:\n",
+    "    raise ValueError(\"No app specified\") from e"
    ]
   },
   {

tests_notebooks/conftest.py

@@ -0,0 +1,130 @@
+import os
+from pathlib import Path
+from time import sleep
+from urllib.parse import urljoin
+
+import pytest
+import requests
+import selenium.webdriver.support.expected_conditions as ec
+from requests.exceptions import ConnectionError
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.wait import WebDriverWait
+
+
+def is_responsive(url):
+    try:
+        response = requests.get(url)
+        if response.status_code == 200:
+            return True
+    except ConnectionError:
+        return False
+
+
+@pytest.fixture(scope="session")
+def screenshot_dir():
+    sdir = Path.joinpath(Path.cwd(), "screenshots")
+    try:
+        os.mkdir(sdir)
+    except FileExistsError:
+        pass
+    return sdir
+
+
+@pytest.fixture(scope="session")
+def docker_compose_file(pytestconfig):
+    return str(Path(pytestconfig.rootdir) / "tests_notebooks" / "docker-compose.yml")
+
+
+@pytest.fixture(scope="session")
+def docker_compose(docker_services):
+    return docker_services._docker_compose
+
+
+@pytest.fixture(scope="session")
+def aiidalab_exec(docker_compose):
+    def execute(command, user=None, **kwargs):
+        workdir = "/home/jovyan/apps/home"
+        if user:
+            command = f"exec --workdir {workdir} -T --user={user} aiidalab {command}"
+        else:
+            command = f"exec --workdir {workdir} -T aiidalab {command}"
+
+        return docker_compose.execute(command, **kwargs)
+
+    return execute
+
+
+@pytest.fixture(scope="session", autouse=True)
+def notebook_service(docker_ip, docker_services, aiidalab_exec):
+    """Ensure that HTTP service is up and responsive."""
+    # Directory ~/apps/home/ is mounted by docker,
+    # make it writeable for jovyan user, needed for `pip install`
+    aiidalab_exec("chmod -R a+rw /home/jovyan/apps/home", user="root")
+
+    aiidalab_exec("pip install --no-cache-dir .")
+
+    # `port_for` takes a container port and returns the corresponding host port
+    port = docker_services.port_for("aiidalab", 8888)
+    url = f"http://{docker_ip}:{port}"
+    token = os.environ["JUPYTER_TOKEN"]
+    docker_services.wait_until_responsive(
+        timeout=30.0, pause=0.1, check=lambda: is_responsive(url)
+    )
+    return url, token
+
+
+@pytest.fixture(scope="function")
+def selenium_driver(selenium, notebook_service):
+    def _selenium_driver(nb_path, url_params=None):
+        url, token = notebook_service
+        url_with_token = urljoin(url, f"apps/apps/home/{nb_path}?token={token}")
+        if url_params is not None:
+            for key, value in url_params.items():
+                url_with_token += f"&{key}={value}"
+        selenium.get(f"{url_with_token}")
+        # By default, let's allow selenium functions to retry for 60s
+        # till a given element is loaded, see:
+        # https://selenium-python.readthedocs.io/waits.html#implicit-waits
+        selenium.implicitly_wait(90)
+        window_width = 800
+        window_height = 600
+        selenium.set_window_size(window_width, window_height)
+
+        selenium.find_element(By.ID, "ipython-main-app")
+        selenium.find_element(By.ID, "notebook-container")
+        selenium.find_element(By.ID, "appmode-busy")
+        # We wait until the appmode spinner disappears. However,
+        # this does not seem to be fully robust, as the spinner might flash
+        # while the page is still loading. So we add explicit sleep here as well.
+        WebDriverWait(selenium, 240).until(
+            ec.invisibility_of_element((By.ID, "appmode-busy"))
+        )
+        sleep(5)
+
+        return selenium
+
+    return _selenium_driver
+
+
+@pytest.fixture
+def final_screenshot(request, screenshot_dir, selenium):
+    """Take screenshot at the end of the test.
+    Screenshot name is generated from the test function name
+    by stripping the 'test_' prefix
+    """
+    screenshot_name = f"{request.function.__name__[5:]}.png"
+    screenshot_path = Path.joinpath(screenshot_dir, screenshot_name)
+    yield
+    selenium.get_screenshot_as_file(screenshot_path)
+
+
+@pytest.fixture
+def firefox_options(firefox_options):
+    firefox_options.add_argument("--headless")
+    return firefox_options
+
+
+@pytest.fixture
+def chrome_options(chrome_options):
+    chrome_options.add_argument("--headless")
+    return chrome_options

tests_notebooks/docker-compose.yml

@@ -0,0 +1,16 @@
+---
+services:
+
+    aiidalab:
+        image: ghcr.io/aiidalab/full-stack:${TAG:-latest}
+        environment:
+            RMQHOST: messaging
+            TZ: Europe/Zurich
+            DOCKER_STACKS_JUPYTER_CMD: notebook
+            SETUP_DEFAULT_AIIDA_PROFILE: 'true'
+            AIIDALAB_DEFAULT_APPS: ''
+            JUPYTER_TOKEN: ${JUPYTER_TOKEN}
+        volumes:
+            - ..:/home/jovyan/apps/home
+        ports:
+            - 8998:8888

tests_notebooks/test_manage_app.py

@@ -0,0 +1,9 @@
+from selenium.webdriver.common.by import By
+
+
+def test_single_app(selenium_driver, final_screenshot):
+    url_params = {"app": "aiidalab-widgets-base"}
+    selenium = selenium_driver("single_app.ipynb", url_params)
+    selenium.set_window_size(1000, 1100)
+    selenium.find_element(By.XPATH, "//button[contains(.,'Uninstall')]")
+    selenium.find_element(By.XPATH, "//button[contains(.,'Install')]")

tests/test_start.py → tests_notebooks/test_start.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 import time
 from contextlib import contextmanager
 
@@ -14,13 +13,14 @@ def get_new_windows(selenium, timeout=2):
     handles.update(set(selenium.window_handles).difference(wh_before))
 
 
-def test_click_appstore(selenium, url):
-    selenium.get(url("apps/apps/home/start.ipynb"))
+def test_click_appstore(selenium_driver, final_screenshot):
+    selenium = selenium_driver("start.ipynb")
     with get_new_windows(selenium) as handles:
         selenium.find_element(By.CSS_SELECTOR, ".fa-puzzle-piece").click()
     assert len(handles) == 1
     selenium.switch_to.window(handles.pop())
     time.sleep(5)
+    selenium.set_window_size(1000, 1100)
     dropdown = selenium.find_element(
         By.XPATH,
         "//div[@id='notebook-container']/div[5]/div[2]/div[2]/div/div[3]/div/div[2]/div/select",
@@ -29,28 +29,28 @@ def test_click_appstore(selenium, url):
     selenium.find_element(By.CSS_SELECTOR, ".widget-button:nth-child(1)").click()
     selenium.find_element(By.CSS_SELECTOR, ".widget-html-content > h1").click()
     time.sleep(5)
-    selenium.get_screenshot_as_file("screenshots/app-store.png")
 
 
-def test_click_help(selenium, url):
-    selenium.get(url("apps/apps/home/start.ipynb"))
+def test_click_help(selenium_driver, final_screenshot):
+    selenium = selenium_driver("start.ipynb")
     selenium.set_window_size(1200, 941)
     with get_new_windows(selenium) as handles:
         selenium.find_element(By.CSS_SELECTOR, ".fa-question").click()
     assert len(handles) == 1
+    # Redirect to https://aiidalab.readthedocs.io
     selenium.switch_to.window(handles.pop())
-    selenium.find_element(By.CSS_SELECTOR, ".mr-md-2").click()
-    selenium.get_screenshot_as_file("screenshots/help.png")
+    # TODO: Instead of selecting a specific element on the Docs page,
+    # validate the URL.
+    # selenium.find_element(By.CSS_SELECTOR, ".mr-md-2").click()
 
 
-def test_click_filemanager(selenium, url):
-    selenium.get(url("apps/apps/home/start.ipynb"))
-    selenium.set_window_size(1200, 941)
+def test_click_filemanager(selenium_driver, final_screenshot):
+    selenium = selenium_driver("start.ipynb")
+    selenium.set_window_size(1000, 941)
     with get_new_windows(selenium) as handles:
         selenium.find_element(By.CSS_SELECTOR, ".fa-file-text-o").click()
     assert len(handles) == 1
     selenium.switch_to.window(handles.pop())
     selenium.find_element(By.LINK_TEXT, "Running").click()
     selenium.find_element(By.LINK_TEXT, "Clusters").click()
     selenium.find_element(By.LINK_TEXT, "Files").click()
-    selenium.get_screenshot_as_file("screenshots/file-manager.png")

tests/test_terminal.py → tests_notebooks/test_terminal.py

@@ -1,12 +1,11 @@
-#!/usr/bin/env python
 from time import sleep
 
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 
 
-def test_terminal(selenium, url):
-    selenium.get(url("apps/apps/home/start.ipynb"))
+def test_terminal(selenium_driver, final_screenshot):
+    selenium = selenium_driver("start.ipynb")
     selenium.set_window_size(1575, 907)
     selenium.find_element(By.CSS_SELECTOR, ".fa-terminal").click()
     page = selenium.window_handles[-1]
@@ -19,5 +18,4 @@ def test_terminal(selenium, url):
     selenium.find_element(By.CSS_SELECTOR, ".xterm-helper-textarea").send_keys(
         Keys.ENTER
     )
-    sleep(1)
-    selenium.get_screenshot_as_file("screenshots/aiidalab-terminal.png")
+    sleep(2)