Refactor to support pyinstaller

MANIFEST.in

@@ -1,2 +1,3 @@
 exclude tests/**
 exclude docs/**
+exclude dev/**

README.md

@@ -40,11 +40,4 @@ Details of how to create a new project are provided in the accompanying [documen
 
 ## Development
 
-If you have cloned the repository, it is recommended to install into a virtual environment. We use [uv](https://docs.astral.sh/uv/) to manage virtual environments. First, [install `uv`](https://docs.astral.sh/uv/getting-started/installation/), then activate the virtual environment:
-
-```bash
-uv sync
-. .venv/bin/activate
-```
-
-To launch the dashboard in development mode run `python -m InsightBoard`.
+See the [development](dev) pages for more information.

dev/README.md

@@ -0,0 +1,30 @@
+# Developer guide
+
+Please consult the [documentation](https://insightboard.readthedocs.io/en/latest) for general usage.
+
+## Virtual environment
+
+We use [uv](https://docs.astral.sh/uv/) to manage virtual environments. First, [install `uv`](https://docs.astral.sh/uv/getting-started/installation/), then activate the virtual environment:
+
+```bash
+uv sync --all-extras
+. .venv/bin/activate
+```
+
+The package will install a script allowing the production server to be launched with `InsightBoard`. To launch the dashboard manually run `python -m InsightBoard`. To launch the dashboard in debug-mode (which uses Dash's bundled Flash server) run `python -m InsightBoard --debug`. If you wish to launch `InsightBoard` from within python, `import InsightBoard` then call `InsightBoard.main()`.
+
+## Versioning
+
+Use [Semantic Versioning](https://semver.org/). Version management is handled by [setuptools-scm](https://setuptools-scm.readthedocs.io/en/latest/), which extracts the current version number from the git tags.
+
+Note that `setuptools-scm` writes to a `src/InsightBoard/version.py` file which _should not_ be checked in to git (the file is listed in `.gitignore`), but will allow packaged releases to query `InsightBoard.__version__` to get the current version number. During development this will look peculiar (e.g. `0.1.0.dev...`).
+
+## Making a release
+
+To create a new release, navigate to github - Releases, then select `Draft a new release`. Create a new tag in the format `vX.Y.Z` (e.g. `v0.1.0`), click `Generate release notes`, then `Publish release`. This will trigger a github action that will build the Python wheels and upload the package to PyPI. It is worth keeping an eye on this action to ensure it completes successfully, and making any necessary changes if it does not.
+
+## Building as an application
+
+_Experimental_
+
+To build the application as a standalone executable, we use [PyInstaller](https://www.pyinstaller.org/). This will create a `dist/InsightBoard` folder containing the executable and all necessary dependencies. To build the application, run `./dev/build_app.sh`.

dev/app.py

@@ -0,0 +1,3 @@
+import InsightBoard
+
+InsightBoard.main()

dev/build_app.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+# Change to the parent directory
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+pushd "$SCRIPT_DIR/.."
+
+# Cleanup any previous build
+rm -rf build dist
+
+# Active the virtual environment (with all dependencies installed)
+uv sync
+uv pip install "adtl[parquet] @ git+https://github.com/globaldothealth/adtl"
+uv pip install pyinstaller
+source .venv/bin/activate
+
+# Build the app
+uv run pyinstaller  \
+    --name InsightBoard \
+    --noconfirm \
+    --hidden-import InsightBoard \
+    --hidden-import InsightBoard.parsers \
+    --add-data "src/InsightBoard/pages/*:./pages/" \
+    src/InsightBoard/__main__.py
+
+# Return to original directory
+popd

src/InsightBoard/__init__.py

@@ -2,25 +2,37 @@
 import time
 import socket
 import logging
-import subprocess
+import threading
 import webbrowser
-from .app import app  # noqa: F401
+import subprocess
 
-from .version import __version__  # noqa: F401
+from waitress import serve
+
+from InsightBoard.app import app  # noqa: F401
+from InsightBoard.version import __version__  # noqa: F401
 
 INSIGHTBOARD_HOST = os.getenv("INSIGHTBOARD_HOST", "127.0.0.1")
 INSIGHTBOARD_PORT = os.getenv("INSIGHTBOARD_PORT", 8050)
 INSIGHTBOARD_TIMEOUT = os.getenv("INSIGHTBOARD_TIMEOUT", 30)
 
 
-def launch_app() -> subprocess.Popen:
+def launch_app():
     logging.getLogger("waitress.queue").setLevel(logging.ERROR)
-    cmd = [
-        "waitress-serve",
-        f"--listen=0.0.0.0:{INSIGHTBOARD_PORT}",
-        "InsightBoard.app:server",
-    ]
-    return subprocess.Popen(cmd)
+    serve(app.server, host=INSIGHTBOARD_HOST, port=INSIGHTBOARD_PORT)
+
+
+def launch_subprocess():
+    logging.getLogger("waitress.queue").setLevel(logging.ERROR)
+    return subprocess.Popen(
+        [
+            "waitress-serve",
+            "--host",
+            INSIGHTBOARD_HOST,
+            "--port",
+            str(INSIGHTBOARD_PORT),
+            "InsightBoard.app:app.server",
+        ]
+    )
 
 
 def check_port(host: str, port: int) -> bool:
@@ -44,16 +56,20 @@ def wait_for_server(
     raise TimeoutError(f"Server did not start within {timeout} seconds")
 
 
-def main(debug=False):
+def main():
     if check_port(INSIGHTBOARD_HOST, INSIGHTBOARD_PORT):
         logging.info("Port is already in use. Opening browser.")
         webbrowser.open(f"http://{INSIGHTBOARD_HOST}:{INSIGHTBOARD_PORT}")
         return
-    process = launch_app()
+    # Launch the server in a separate thread
+    server_thread = threading.Thread(target=launch_app)
+    server_thread.start()
+    # Wait for startup before opening web browser
     wait_for_server(
         INSIGHTBOARD_HOST,
         INSIGHTBOARD_PORT,
         INSIGHTBOARD_TIMEOUT,
     )
     webbrowser.open(f"http://{INSIGHTBOARD_HOST}:{INSIGHTBOARD_PORT}")
-    process.wait()
+    # Join the server thread and wait for it to finish or be closed
+    server_thread.join()

src/InsightBoard/__main__.py

@@ -1,5 +1,16 @@
-from .app import app
+import argparse
+from InsightBoard import main
+from InsightBoard.app import app
 
 
 if __name__ == "__main__":
-    app.run(debug=True)  # pragma: no cover
+    p = argparse.ArgumentParser()
+    p.add_argument("--debug", action="store_true")
+    args = p.parse_args()
+
+    if args.debug:
+        # Dash launches a Flask development server
+        app.run(debug=True)
+    else:
+        # Otherwise, launch the production server
+        main()

src/InsightBoard/app.py

@@ -1,10 +1,22 @@
-from pathlib import Path
+import sys
 import dash
-from dash import dcc, html, Input, Output
 import dash_bootstrap_components as dbc
 
-from .config import ConfigManager
-from .utils import get_projects_list, get_default_project, get_custom_assets_folder
+from dash import dcc, html, Input, Output
+from pathlib import Path
+
+from InsightBoard.config import ConfigManager
+from InsightBoard.utils import (
+    get_projects_list,
+    get_default_project,
+    get_custom_assets_folder,
+)
+
+# If running from PyInstaller, get the path to the temporary directory
+if hasattr(sys, "_MEIPASS"):
+    base_path = Path(sys._MEIPASS)
+else:
+    base_path = Path(__file__).parent
 
 projects = get_projects_list()
 default_project = get_default_project()
@@ -21,10 +33,12 @@
     else []
 )
 cogwheel = (html.I(className="fas fa-cog"),)
+pages_path = base_path / "pages"
 
 app = dash.Dash(
     __name__,
     use_pages=True,
+    pages_folder=str(pages_path),
     external_stylesheets=[
         dbc.themes.BOOTSTRAP,
         (
@@ -83,8 +97,9 @@ def ProjectDropDown():
 @app.callback(Output("project", "data"), Input("project-dropdown", "value"))
 def store_selected_project(project):
     config = ConfigManager()
-    config.set("project.default", project)
-    config.save()
+    if project:
+        config.set("project.default", project)
+        config.save()
     return project
 
 

src/InsightBoard/config/__init__.py

@@ -1 +1 @@
-from .config import ConfigManager  # noqa: F401
+from InsightBoard.config.config import ConfigManager  # noqa: F401

src/InsightBoard/database/__init__.py

@@ -1 +1,6 @@
-from .database import Database, DatabaseBackend, WritePolicy, BackupPolicy  # noqa: F401
+from InsightBoard.database.database import (  # noqa: F401
+    Database,
+    DatabaseBackend,
+    WritePolicy,
+    BackupPolicy,
+)

src/InsightBoard/project/__init__.py

@@ -1 +1 @@
-from .project import Project  # noqa: F401
+from InsightBoard.project.project import Project  # noqa: F401

src/InsightBoard/utils.py

@@ -6,8 +6,8 @@
 from pathlib import Path
 from jsonschema import Draft7Validator
 
-from .project import Project
-from .project.project import (  # expose downstream # noqa: F401
+from InsightBoard.project import Project
+from InsightBoard.project.project import (  # expose downstream # noqa: F401
     get_projects_list,
     get_default_project,
     get_custom_assets_folder,

tests/system/InsightBoard/projects/sample_project/parsers/adtl-source1.py

@@ -1,7 +1,5 @@
-from tempfile import NamedTemporaryFile
 import pandas as pd
 from pathlib import Path
-import adtl
 from InsightBoard.parsers import parse_adtl
 
 SPECIFICATION_FILE = Path("adtl") / "source1.toml"

tests/system/test_e2e.py

@@ -1,19 +1,21 @@
 import time
 import pytest
 from pathlib import Path
-from utils import (
+from utils import (  # noqa: F401 (linter does not recognise driver as a fixture)
     driver,
     page_upload,
+    select_project,
     chromedriver_present,
     save_screenshot,
 )
 
 
 @pytest.mark.skipif(not chromedriver_present, reason="chromedriver not present")
-def test_insightboard(driver):
-    upload = page_upload(driver)
-    upload.clear_data()
+def test_insightboard(driver):  # noqa: F811 (driver is a fixture)
     try:
+        select_project(driver, "sample_project")
+        upload = page_upload(driver)
+        upload.clear_data()
         upload.select_parser("adtl-source1")
         data_file = (
             Path(__file__).parent

tests/system/utils.py

@@ -28,6 +28,16 @@ def save_screenshot(driver, name="screenshot"):
     return str(screenshot_path)
 
 
+def timeout(driver, fcn):
+    for _ in range(10):
+        try:
+            fcn()
+            break
+        except Exception:
+            time.sleep(1)
+    assert fcn()
+
+
 @pytest.fixture
 def driver():
     service = Service(binary_path)
@@ -51,30 +61,24 @@ def driver():
         f.write(tomli_w.dumps(new_config))
 
     # Launch InsightBoard and wait for server to start
-    process = InsightBoard.launch_app()
-    InsightBoard.wait_for_server()
+    process = InsightBoard.launch_subprocess()  # app initializes during import ...
+    InsightBoard.wait_for_server()  # ... subprocess isolates startup
 
     # Open the Dash app in the browser
     driver.get("http://127.0.0.1:8050")
-    for _ in range(10):
-        try:
-            driver.find_element(By.TAG_NAME, "h1")
-            break
-        except Exception:
-            time.sleep(1)
-
-    # Stat with the sample project selected
-    select_project(driver, "sample_project")
+    WebDriverWait(driver, 10).until(
+        EC.visibility_of_element_located((By.ID, "project-dropdown"))
+    )
 
     yield driver
 
     # Restore the InsightBoard config file and close chromedriver / Dash app
     driver.quit()
+    process.terminate()
+    process.wait()
     if config:
         with open(config_file, "w") as f:
             f.write(tomli_w.dumps(config))
-    process.kill()
-    process.wait()
 
 
 def select_project(driver, project_name):
@@ -87,15 +91,20 @@ def page_upload(driver):
     )
     upload_link.click()
     # assert that the upload page is loaded
-    time.sleep(1)
-    assert driver.find_element(By.TAG_NAME, "h1").text == "Upload data"
+    timeout(
+        driver, lambda: driver.find_element(By.TAG_NAME, "h1").text == "Upload data"
+    )
     return PageUpload(driver)
 
 
 def dropdown_select(driver, dropdown_id, option):
     dropdown = driver.find_element(By.ID, dropdown_id)
     dropdown.click()
-    option_to_select = dropdown.find_element(By.XPATH, f'//div[text()="{option}"]')
+    option_to_select = WebDriverWait(driver, 10).until(
+        EC.visibility_of_element_located(
+            (By.XPATH, f'//div[@id="{dropdown_id}"]//div[text()="{option}"]')
+        )
+    )
     option_to_select.click()
     dropdown_value = dropdown.find_element(By.CLASS_NAME, "Select-value-label")
     assert dropdown_value.text == option

tests/unit/pages/test_upload.py

@@ -5,21 +5,21 @@
     update_table,
     remove_quotes,
     clean_value,
-    update_edited_data,
-    error_report_message,
-    text_to_html,
-    errorlist_to_sentence,
-    errors_to_dict,
-    validate_errors,
-    validate_log,
-    ctx_trigger,
-    parse_file,
-    parse_data,
-    highlight_and_tooltip_changes,
-    update_table_style_and_validate,
-    download_csv,
-    display_confirm_dialog,
-    commit_to_database,
+    # update_edited_data,
+    # error_report_message,
+    # text_to_html,
+    # errorlist_to_sentence,
+    # errors_to_dict,
+    # validate_errors,
+    # validate_log,
+    # ctx_trigger,
+    # parse_file,
+    # parse_data,
+    # highlight_and_tooltip_changes,
+    # update_table_style_and_validate,
+    # download_csv,
+    # display_confirm_dialog,
+    # commit_to_database,
 )