diff --git a/MANIFEST.in b/MANIFEST.in index f0be065..53a9af8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ exclude tests/** exclude docs/** +exclude dev/** diff --git a/README.md b/README.md index ac9f63d..3d776ca 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,4 @@ Details of how to create a new project are provided in the accompanying [documen ## Development -If you have cloned the repository, it is recommended to install into a virtual environment. We use [uv](https://docs.astral.sh/uv/) to manage virtual environments. First, [install `uv`](https://docs.astral.sh/uv/getting-started/installation/), then activate the virtual environment: - -```bash -uv sync -. .venv/bin/activate -``` - -To launch the dashboard in development mode run `python -m InsightBoard`. +See the [development](dev) pages for more information. diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 0000000..fa7e2ee --- /dev/null +++ b/dev/README.md @@ -0,0 +1,30 @@ +# Developer guide + +Please consult the [documentation](https://insightboard.readthedocs.io/en/latest) for general usage. + +## Virtual environment + +We use [uv](https://docs.astral.sh/uv/) to manage virtual environments. First, [install `uv`](https://docs.astral.sh/uv/getting-started/installation/), then activate the virtual environment: + +```bash +uv sync --all-extras +. .venv/bin/activate +``` + +The package will install a script allowing the production server to be launched with `InsightBoard`. To launch the dashboard manually run `python -m InsightBoard`. To launch the dashboard in debug-mode (which uses Dash's bundled Flash server) run `python -m InsightBoard --debug`. If you wish to launch `InsightBoard` from within python, `import InsightBoard` then call `InsightBoard.main()`. + +## Versioning + +Use [Semantic Versioning](https://semver.org/). Version management is handled by [setuptools-scm](https://setuptools-scm.readthedocs.io/en/latest/), which extracts the current version number from the git tags. + +Note that `setuptools-scm` writes to a `src/InsightBoard/version.py` file which _should not_ be checked in to git (the file is listed in `.gitignore`), but will allow packaged releases to query `InsightBoard.__version__` to get the current version number. During development this will look peculiar (e.g. `0.1.0.dev...`). + +## Making a release + +To create a new release, navigate to github - Releases, then select `Draft a new release`. Create a new tag in the format `vX.Y.Z` (e.g. `v0.1.0`), click `Generate release notes`, then `Publish release`. This will trigger a github action that will build the Python wheels and upload the package to PyPI. It is worth keeping an eye on this action to ensure it completes successfully, and making any necessary changes if it does not. + +## Building as an application + +_Experimental_ + +To build the application as a standalone executable, we use [PyInstaller](https://www.pyinstaller.org/). This will create a `dist/InsightBoard` folder containing the executable and all necessary dependencies. To build the application, run `./dev/build_app.sh`. diff --git a/dev/app.py b/dev/app.py new file mode 100644 index 0000000..0fd2491 --- /dev/null +++ b/dev/app.py @@ -0,0 +1,3 @@ +import InsightBoard + +InsightBoard.main() diff --git a/dev/build_app.sh b/dev/build_app.sh new file mode 100755 index 0000000..29374bd --- /dev/null +++ b/dev/build_app.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +# Change to the parent directory +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd "$SCRIPT_DIR/.." + +# Cleanup any previous build +rm -rf build dist + +# Active the virtual environment (with all dependencies installed) +uv sync +uv pip install "adtl[parquet] @ git+https://github.com/globaldothealth/adtl" +uv pip install pyinstaller +source .venv/bin/activate + +# Build the app +uv run pyinstaller \ + --name InsightBoard \ + --noconfirm \ + --hidden-import InsightBoard \ + --hidden-import InsightBoard.parsers \ + --add-data "src/InsightBoard/pages/*:./pages/" \ + src/InsightBoard/__main__.py + +# Return to original directory +popd diff --git a/src/InsightBoard/__init__.py b/src/InsightBoard/__init__.py index 1f42c90..253b76f 100644 --- a/src/InsightBoard/__init__.py +++ b/src/InsightBoard/__init__.py @@ -2,25 +2,37 @@ import time import socket import logging -import subprocess +import threading import webbrowser -from .app import app # noqa: F401 +import subprocess -from .version import __version__ # noqa: F401 +from waitress import serve + +from InsightBoard.app import app # noqa: F401 +from InsightBoard.version import __version__ # noqa: F401 INSIGHTBOARD_HOST = os.getenv("INSIGHTBOARD_HOST", "127.0.0.1") INSIGHTBOARD_PORT = os.getenv("INSIGHTBOARD_PORT", 8050) INSIGHTBOARD_TIMEOUT = os.getenv("INSIGHTBOARD_TIMEOUT", 30) -def launch_app() -> subprocess.Popen: +def launch_app(): logging.getLogger("waitress.queue").setLevel(logging.ERROR) - cmd = [ - "waitress-serve", - f"--listen=0.0.0.0:{INSIGHTBOARD_PORT}", - "InsightBoard.app:server", - ] - return subprocess.Popen(cmd) + serve(app.server, host=INSIGHTBOARD_HOST, port=INSIGHTBOARD_PORT) + + +def launch_subprocess(): + logging.getLogger("waitress.queue").setLevel(logging.ERROR) + return subprocess.Popen( + [ + "waitress-serve", + "--host", + INSIGHTBOARD_HOST, + "--port", + str(INSIGHTBOARD_PORT), + "InsightBoard.app:app.server", + ] + ) def check_port(host: str, port: int) -> bool: @@ -44,16 +56,20 @@ def wait_for_server( raise TimeoutError(f"Server did not start within {timeout} seconds") -def main(debug=False): +def main(): if check_port(INSIGHTBOARD_HOST, INSIGHTBOARD_PORT): logging.info("Port is already in use. Opening browser.") webbrowser.open(f"http://{INSIGHTBOARD_HOST}:{INSIGHTBOARD_PORT}") return - process = launch_app() + # Launch the server in a separate thread + server_thread = threading.Thread(target=launch_app) + server_thread.start() + # Wait for startup before opening web browser wait_for_server( INSIGHTBOARD_HOST, INSIGHTBOARD_PORT, INSIGHTBOARD_TIMEOUT, ) webbrowser.open(f"http://{INSIGHTBOARD_HOST}:{INSIGHTBOARD_PORT}") - process.wait() + # Join the server thread and wait for it to finish or be closed + server_thread.join() diff --git a/src/InsightBoard/__main__.py b/src/InsightBoard/__main__.py index 6e824cd..9061b41 100644 --- a/src/InsightBoard/__main__.py +++ b/src/InsightBoard/__main__.py @@ -1,5 +1,16 @@ -from .app import app +import argparse +from InsightBoard import main +from InsightBoard.app import app if __name__ == "__main__": - app.run(debug=True) # pragma: no cover + p = argparse.ArgumentParser() + p.add_argument("--debug", action="store_true") + args = p.parse_args() + + if args.debug: + # Dash launches a Flask development server + app.run(debug=True) + else: + # Otherwise, launch the production server + main() diff --git a/src/InsightBoard/app.py b/src/InsightBoard/app.py index 0d0e651..233266c 100644 --- a/src/InsightBoard/app.py +++ b/src/InsightBoard/app.py @@ -1,10 +1,22 @@ -from pathlib import Path +import sys import dash -from dash import dcc, html, Input, Output import dash_bootstrap_components as dbc -from .config import ConfigManager -from .utils import get_projects_list, get_default_project, get_custom_assets_folder +from dash import dcc, html, Input, Output +from pathlib import Path + +from InsightBoard.config import ConfigManager +from InsightBoard.utils import ( + get_projects_list, + get_default_project, + get_custom_assets_folder, +) + +# If running from PyInstaller, get the path to the temporary directory +if hasattr(sys, "_MEIPASS"): + base_path = Path(sys._MEIPASS) +else: + base_path = Path(__file__).parent projects = get_projects_list() default_project = get_default_project() @@ -21,10 +33,12 @@ else [] ) cogwheel = (html.I(className="fas fa-cog"),) +pages_path = base_path / "pages" app = dash.Dash( __name__, use_pages=True, + pages_folder=str(pages_path), external_stylesheets=[ dbc.themes.BOOTSTRAP, ( @@ -83,8 +97,9 @@ def ProjectDropDown(): @app.callback(Output("project", "data"), Input("project-dropdown", "value")) def store_selected_project(project): config = ConfigManager() - config.set("project.default", project) - config.save() + if project: + config.set("project.default", project) + config.save() return project diff --git a/src/InsightBoard/config/__init__.py b/src/InsightBoard/config/__init__.py index 4af16e5..f4070dd 100644 --- a/src/InsightBoard/config/__init__.py +++ b/src/InsightBoard/config/__init__.py @@ -1 +1 @@ -from .config import ConfigManager # noqa: F401 +from InsightBoard.config.config import ConfigManager # noqa: F401 diff --git a/src/InsightBoard/database/__init__.py b/src/InsightBoard/database/__init__.py index bdf7f57..2359624 100644 --- a/src/InsightBoard/database/__init__.py +++ b/src/InsightBoard/database/__init__.py @@ -1 +1,6 @@ -from .database import Database, DatabaseBackend, WritePolicy, BackupPolicy # noqa: F401 +from InsightBoard.database.database import ( # noqa: F401 + Database, + DatabaseBackend, + WritePolicy, + BackupPolicy, +) diff --git a/src/InsightBoard/project/__init__.py b/src/InsightBoard/project/__init__.py index a2d7927..0fcc07d 100644 --- a/src/InsightBoard/project/__init__.py +++ b/src/InsightBoard/project/__init__.py @@ -1 +1 @@ -from .project import Project # noqa: F401 +from InsightBoard.project.project import Project # noqa: F401 diff --git a/src/InsightBoard/utils.py b/src/InsightBoard/utils.py index 1647011..ba9fa84 100644 --- a/src/InsightBoard/utils.py +++ b/src/InsightBoard/utils.py @@ -6,8 +6,8 @@ from pathlib import Path from jsonschema import Draft7Validator -from .project import Project -from .project.project import ( # expose downstream # noqa: F401 +from InsightBoard.project import Project +from InsightBoard.project.project import ( # expose downstream # noqa: F401 get_projects_list, get_default_project, get_custom_assets_folder, diff --git a/tests/system/InsightBoard/projects/sample_project/parsers/adtl-source1.py b/tests/system/InsightBoard/projects/sample_project/parsers/adtl-source1.py index a196363..0a4bfb4 100644 --- a/tests/system/InsightBoard/projects/sample_project/parsers/adtl-source1.py +++ b/tests/system/InsightBoard/projects/sample_project/parsers/adtl-source1.py @@ -1,7 +1,5 @@ -from tempfile import NamedTemporaryFile import pandas as pd from pathlib import Path -import adtl from InsightBoard.parsers import parse_adtl SPECIFICATION_FILE = Path("adtl") / "source1.toml" diff --git a/tests/system/test_e2e.py b/tests/system/test_e2e.py index a314d07..4ac463d 100644 --- a/tests/system/test_e2e.py +++ b/tests/system/test_e2e.py @@ -1,19 +1,21 @@ import time import pytest from pathlib import Path -from utils import ( +from utils import ( # noqa: F401 (linter does not recognise driver as a fixture) driver, page_upload, + select_project, chromedriver_present, save_screenshot, ) @pytest.mark.skipif(not chromedriver_present, reason="chromedriver not present") -def test_insightboard(driver): - upload = page_upload(driver) - upload.clear_data() +def test_insightboard(driver): # noqa: F811 (driver is a fixture) try: + select_project(driver, "sample_project") + upload = page_upload(driver) + upload.clear_data() upload.select_parser("adtl-source1") data_file = ( Path(__file__).parent diff --git a/tests/system/utils.py b/tests/system/utils.py index 40682b2..c3bdee9 100644 --- a/tests/system/utils.py +++ b/tests/system/utils.py @@ -28,6 +28,16 @@ def save_screenshot(driver, name="screenshot"): return str(screenshot_path) +def timeout(driver, fcn): + for _ in range(10): + try: + fcn() + break + except Exception: + time.sleep(1) + assert fcn() + + @pytest.fixture def driver(): service = Service(binary_path) @@ -51,30 +61,24 @@ def driver(): f.write(tomli_w.dumps(new_config)) # Launch InsightBoard and wait for server to start - process = InsightBoard.launch_app() - InsightBoard.wait_for_server() + process = InsightBoard.launch_subprocess() # app initializes during import ... + InsightBoard.wait_for_server() # ... subprocess isolates startup # Open the Dash app in the browser driver.get("http://127.0.0.1:8050") - for _ in range(10): - try: - driver.find_element(By.TAG_NAME, "h1") - break - except Exception: - time.sleep(1) - - # Stat with the sample project selected - select_project(driver, "sample_project") + WebDriverWait(driver, 10).until( + EC.visibility_of_element_located((By.ID, "project-dropdown")) + ) yield driver # Restore the InsightBoard config file and close chromedriver / Dash app driver.quit() + process.terminate() + process.wait() if config: with open(config_file, "w") as f: f.write(tomli_w.dumps(config)) - process.kill() - process.wait() def select_project(driver, project_name): @@ -87,15 +91,20 @@ def page_upload(driver): ) upload_link.click() # assert that the upload page is loaded - time.sleep(1) - assert driver.find_element(By.TAG_NAME, "h1").text == "Upload data" + timeout( + driver, lambda: driver.find_element(By.TAG_NAME, "h1").text == "Upload data" + ) return PageUpload(driver) def dropdown_select(driver, dropdown_id, option): dropdown = driver.find_element(By.ID, dropdown_id) dropdown.click() - option_to_select = dropdown.find_element(By.XPATH, f'//div[text()="{option}"]') + option_to_select = WebDriverWait(driver, 10).until( + EC.visibility_of_element_located( + (By.XPATH, f'//div[@id="{dropdown_id}"]//div[text()="{option}"]') + ) + ) option_to_select.click() dropdown_value = dropdown.find_element(By.CLASS_NAME, "Select-value-label") assert dropdown_value.text == option diff --git a/tests/unit/pages/test_upload.py b/tests/unit/pages/test_upload.py index e8cae2e..5560bd0 100644 --- a/tests/unit/pages/test_upload.py +++ b/tests/unit/pages/test_upload.py @@ -5,21 +5,21 @@ update_table, remove_quotes, clean_value, - update_edited_data, - error_report_message, - text_to_html, - errorlist_to_sentence, - errors_to_dict, - validate_errors, - validate_log, - ctx_trigger, - parse_file, - parse_data, - highlight_and_tooltip_changes, - update_table_style_and_validate, - download_csv, - display_confirm_dialog, - commit_to_database, + # update_edited_data, + # error_report_message, + # text_to_html, + # errorlist_to_sentence, + # errors_to_dict, + # validate_errors, + # validate_log, + # ctx_trigger, + # parse_file, + # parse_data, + # highlight_and_tooltip_changes, + # update_table_style_and_validate, + # download_csv, + # display_confirm_dialog, + # commit_to_database, ) diff --git a/tests/unit/test_insightboard.py b/tests/unit/test_insightboard.py index a6839e9..83be851 100644 --- a/tests/unit/test_insightboard.py +++ b/tests/unit/test_insightboard.py @@ -2,8 +2,6 @@ from unittest.mock import patch -import InsightBoard as ib - def test_main_fcn(): with ( diff --git a/uv.lock b/uv.lock index e4861c5..0f04e1f 100644 --- a/uv.lock +++ b/uv.lock @@ -305,7 +305,7 @@ wheels = [ [[package]] name = "insightboard" -version = "0.0.1.dev70+gebc1735.d20241009" +version = "0.0.1.dev67+g27f9806.d20241010" source = { editable = "." } dependencies = [ { name = "dash" },