From d7295497b5a8efb8b1c00910980b8f64c2132807 Mon Sep 17 00:00:00 2001 From: Antoine Paletta Date: Wed, 22 Nov 2023 13:40:36 +0000 Subject: [PATCH 1/6] feat: modify makefile for debugging jupyter --- Makefile | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 796c941b0..b12f28d14 100644 --- a/Makefile +++ b/Makefile @@ -223,7 +223,7 @@ build-packages-python-standalone: ## Build Python packages (standalone) --workdir=/app/build \ $(docker_development_image_repository):$(docker_image_version) \ /bin/bash -c "cmake -DBUILD_UNIT_TESTS=OFF -DBUILD_PYTHON_BINDINGS=ON .. \ - && $(MAKE) -j 4 \ + && $(MAKE) -j $(shell nproc) \ && mkdir -p /app/packages/python \ && mv /app/build/bindings/python/dist/*.whl /app/packages/python" @@ -237,7 +237,7 @@ start-development-no-link: build-development-image ## Start development environm -it \ --rm \ --privileged \ - --volume="$(CURDIR):/app:delegated" \ + --volume="$(CURDIR):/app" \ --workdir=/app/build \ $(docker_development_image_repository):$(docker_image_version) \ /bin/bash @@ -289,7 +289,20 @@ start-jupyter-notebook: build-release-image-jupyter ## Start Jupyter Notebook en .PHONY: start-jupyter-notebook -debug-jupyter-notebook: build-release-image-jupyter ## Debug jupyter notebook using the ostk-astro package built from current source code, must have run make start-development and ostk-build and ostk-install-python to use this +debug-jupyter-notebook: build-release-image-jupyter ## Debug jupyter notebook using the ostk-astro package built from current source code + + @ echo "Building Python$(jupyter_python_version) packages..." + @ mkdir -p $(CURDIR)/build + @ mkdir -p $(CURDIR)/packages/python + + docker run \ + --rm \ + --volume="$(CURDIR):/app:delegated" \ + --workdir=/app/build \ + $(docker_development_image_repository):$(docker_image_version) \ + /bin/bash -c "cmake -DBUILD_UNIT_TESTS=OFF -DBUILD_BENCHMARK=OFF -DBUILD_PYTHON_BINDINGS=ON -DPYTHON_SEARCH_VERSIONS="$(jupyter_python_version)" .. \ + && $(MAKE) -j $(shell nproc) \ + && cp /app/build/bindings/python/dist/*.whl /app/packages/python" @ echo "Debugging Jupyter Notebook environment..." @@ -298,15 +311,14 @@ debug-jupyter-notebook: build-release-image-jupyter ## Debug jupyter notebook us --rm \ --user=root \ --publish="$(jupyter_notebook_port):8888" \ - --volume="$(CURDIR)/bindings/python/docs:/home/jovyan/docs" \ - --volume="$(CURDIR)/tutorials/python/notebooks:/home/jovyan/tutorials" \ - --volume="$(CURDIR)/build/bindings/python/OpenSpaceToolkit${project_name_camel_case}Py-python-package-$(jupyter_python_version):/opt/conda/lib/python$(jupyter_python_version)/site-packages/ostk/$(project_name)" \ + --volume="$(CURDIR)/bindings/python/docs:/home/jovyan/docs:delegated" \ + --volume="$(CURDIR)/tutorials/python/notebooks:/home/jovyan/tutorials:delegated" \ + --volume="$(CURDIR)/packages/python:/home/jovyan/.packages:delegated" \ --workdir="/home/jovyan" \ $(docker_release_image_jupyter_repository):$(docker_image_version) \ - bash -c "chown -R jovyan:users /home/jovyan ; python$(jupyter_python_version) -m pip install /opt/conda/lib/python$(jupyter_python_version)/site-packages/ostk/$(project_name)/ --force-reinstall ; start-notebook.sh --ServerApp.token=''" + bash -c "chown -R jovyan:users /home/jovyan ; python$(jupyter_python_version) -m pip install /home/jovyan/.packages/*.whl --force-reinstall ; start-notebook.sh --ServerApp.token=''" - bash -c "sudo chown -R $(shell id -u):$(shell id -g) $(CURDIR)/bindings/python/docs" - bash -c "sudo chown -R $(shell id -u):$(shell id -g) $(CURDIR)/tutorials/python/notebooks" + bash -c "sudo chown -R $(shell id -u):$(shell id -g) $(CURDIR)" .PHONY: debug-jupyter-notebook @@ -457,7 +469,7 @@ test-unit-cpp-standalone: ## Run C++ unit tests (standalone) --workdir=/app/build \ $(docker_development_image_repository):$(docker_image_version) \ /bin/bash -c "cmake -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_UNIT_TESTS=ON .. \ - && $(MAKE) -j 4 \ + && $(MAKE) -j $(shell nproc) \ && $(MAKE) test" .PHONY: test-unit-cpp-standalone @@ -480,7 +492,7 @@ test-unit-python-standalone: ## Run Python unit tests (standalone) --entrypoint="" \ $(docker_development_image_repository):$(docker_image_version) \ /bin/bash -c "cmake -DBUILD_PYTHON_BINDINGS=ON -DBUILD_UNIT_TESTS=OFF .. \ - && $(MAKE) -j 4 && python3.11 -m pip install --root-user-action=ignore bindings/python/dist/*311*.whl \ + && $(MAKE) -j $(shell nproc) && python3.11 -m pip install --root-user-action=ignore bindings/python/dist/*311*.whl \ && python3.11 -m pip install plotly pandas \ && python3.11 -m pip install git+https://github.com/lucas-bremond/cesiumpy.git#egg=cesiumpy \ && cd /usr/local/lib/python3.11/site-packages/ostk/$(project_name)/ \ @@ -513,7 +525,7 @@ test-coverage-cpp-standalone: ## Run C++ tests with coverage (standalone) --workdir=/app/build \ $(docker_development_image_repository):$(docker_image_version) \ /bin/bash -c "cmake -DBUILD_UNIT_TESTS=ON -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_CODE_COVERAGE=ON .. \ - && $(MAKE) -j 4 \ + && $(MAKE) -j $(shell nproc) \ && $(MAKE) coverage \ && (rm -rf /app/coverage || true) \ && mkdir /app/coverage \ @@ -546,7 +558,7 @@ benchmark-cpp-standalone: ## Run C++ benchmarks (standalone) --workdir=/app/build \ $(docker_development_image_repository):$(docker_image_version) \ /bin/bash -c "cmake -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_UNIT_TESTS=OFF -DBUILD_BENCHMARK=ON .. \ - && $(MAKE) -j 4 \ + && $(MAKE) -j $(shell nproc) \ && ./../bin/open-space-toolkit-$(project_name).benchmark --benchmark_out_format=json --benchmark_out=./../bin/benchmark_result.json" .PHONY: benchmark-cpp-standalone @@ -556,12 +568,11 @@ clean: ## Clean @ echo "Cleaning up..." rm -rf "$(CURDIR)/build" - rm -rf "$(CURDIR)/bin/"*.test* + rm -rf "$(CURDIR)/bin" rm -rf "$(CURDIR)/docs/html" rm -rf "$(CURDIR)/docs/latex" - rm -rf "$(CURDIR)/lib/"*.so* + rm -rf "$(CURDIR)/lib" rm -rf "$(CURDIR)/coverage" - rm -rf "$(CURDIR)/benchmark" rm -rf "$(CURDIR)/packages" rm -rf "$(CURDIR)/.open-space-toolkit" From e30f4e52cd79d16e9485c0b6a6a463bfa78b1510 Mon Sep 17 00:00:00 2001 From: Antoine Paletta Date: Wed, 22 Nov 2023 14:05:04 +0000 Subject: [PATCH 2/6] fix: add build-dev-image make req to debug-jupyter-image target --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b12f28d14..90227468d 100644 --- a/Makefile +++ b/Makefile @@ -289,13 +289,14 @@ start-jupyter-notebook: build-release-image-jupyter ## Start Jupyter Notebook en .PHONY: start-jupyter-notebook -debug-jupyter-notebook: build-release-image-jupyter ## Debug jupyter notebook using the ostk-astro package built from current source code +debug-jupyter-notebook: build-development-image build-release-image-jupyter ## Debug jupyter notebook using the ostk-astro package built from current source code @ echo "Building Python$(jupyter_python_version) packages..." @ mkdir -p $(CURDIR)/build @ mkdir -p $(CURDIR)/packages/python docker run \ + -it \ --rm \ --volume="$(CURDIR):/app:delegated" \ --workdir=/app/build \ From d1395347b710631828c8278c0955528df1b129f2 Mon Sep 17 00:00:00 2001 From: Antoine Paletta Date: Wed, 22 Nov 2023 15:45:25 +0000 Subject: [PATCH 3/6] fix: remove extra targets at top of makefile --- Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile b/Makefile index 90227468d..4192c0c5a 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,6 @@ project_name := astrodynamics project_version := $(shell git describe --tags --always) -project_major_version := $(shell git describe --tags --always | cut -c 1) docker_registry_path := openspacecollective docker_image_repository := $(docker_registry_path)/open-space-toolkit-$(project_name) @@ -17,8 +16,6 @@ jupyter_notebook_port := 9005 jupyter_notebook_image_repository := jupyter/scipy-notebook:x86_64-python-3.11.3 jupyter_python_version := 3.11 -project_name_camel_case := $(shell echo $(project_name) | sed -r 's/(^|-)([a-z])/\U\2/g') - pull: ## Pull all images @ echo "Pulling images..." From 668b0c94cbbb898091eb1a1e8e8a81e53aec3087 Mon Sep 17 00:00:00 2001 From: Antoine Paletta Date: Thu, 23 Nov 2023 12:35:24 +0000 Subject: [PATCH 4/6] fix: address feedback from vishwa --- Makefile | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 4192c0c5a..efa6655bc 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ docker_release_image_jupyter_repository := $(docker_image_repository)-jupyter jupyter_notebook_port := 9005 jupyter_notebook_image_repository := jupyter/scipy-notebook:x86_64-python-3.11.3 jupyter_python_version := 3.11 +extract_python_package_version := $(shell echo $(project_version) | sed 's/-/./' | sed 's/-.*//') pull: ## Pull all images @@ -220,7 +221,7 @@ build-packages-python-standalone: ## Build Python packages (standalone) --workdir=/app/build \ $(docker_development_image_repository):$(docker_image_version) \ /bin/bash -c "cmake -DBUILD_UNIT_TESTS=OFF -DBUILD_PYTHON_BINDINGS=ON .. \ - && $(MAKE) -j $(shell nproc) \ + && $(MAKE) -j 4 \ && mkdir -p /app/packages/python \ && mv /app/build/bindings/python/dist/*.whl /app/packages/python" @@ -234,7 +235,7 @@ start-development-no-link: build-development-image ## Start development environm -it \ --rm \ --privileged \ - --volume="$(CURDIR):/app" \ + --volume="$(CURDIR):/app:delegated" \ --workdir=/app/build \ $(docker_development_image_repository):$(docker_image_version) \ /bin/bash @@ -251,7 +252,7 @@ start-development-link: build-development-image ## Start linked development envi .PHONY: start-development-link -ifndef link +ifndef links start-development dev: start-development-no-link else start-development dev: start-development-link @@ -270,7 +271,7 @@ start-python: build-release-image-python ## Start Python runtime environment .PHONY: start-python -start-jupyter-notebook: build-release-image-jupyter ## Start Jupyter Notebook environment +start-jupyter: build-release-image-jupyter ## Start Jupyter Notebook environment @ echo "Starting Jupyter Notebook environment..." @@ -286,11 +287,9 @@ start-jupyter-notebook: build-release-image-jupyter ## Start Jupyter Notebook en .PHONY: start-jupyter-notebook -debug-jupyter-notebook: build-development-image build-release-image-jupyter ## Debug jupyter notebook using the ostk-astro package built from current source code +debug-jupyter: build-development-image ## Debug jupyter notebook using the ostk-astro package built from current source code @ echo "Building Python$(jupyter_python_version) packages..." - @ mkdir -p $(CURDIR)/build - @ mkdir -p $(CURDIR)/packages/python docker run \ -it \ @@ -300,7 +299,15 @@ debug-jupyter-notebook: build-development-image build-release-image-jupyter ## D $(docker_development_image_repository):$(docker_image_version) \ /bin/bash -c "cmake -DBUILD_UNIT_TESTS=OFF -DBUILD_BENCHMARK=OFF -DBUILD_PYTHON_BINDINGS=ON -DPYTHON_SEARCH_VERSIONS="$(jupyter_python_version)" .. \ && $(MAKE) -j $(shell nproc) \ - && cp /app/build/bindings/python/dist/*.whl /app/packages/python" + && mkdir -p /app/packages/python \ + && rm -rf /app/packages/python/* \ + && cp /app/build/bindings/python/dist/*$(extract_python_package_version)*.whl /app/packages/python" + + @ $(MAKE) debug-jupyter-standalone + +.PHONY: debug-jupyter + +debug-jupyter-standalone: build-release-image-jupyter ## Debug jupyter notebook using the ostk-astro package built from current source code @ echo "Debugging Jupyter Notebook environment..." @@ -316,9 +323,9 @@ debug-jupyter-notebook: build-development-image build-release-image-jupyter ## D $(docker_release_image_jupyter_repository):$(docker_image_version) \ bash -c "chown -R jovyan:users /home/jovyan ; python$(jupyter_python_version) -m pip install /home/jovyan/.packages/*.whl --force-reinstall ; start-notebook.sh --ServerApp.token=''" - bash -c "sudo chown -R $(shell id -u):$(shell id -g) $(CURDIR)" + @ sudo chown -R $(shell id -u):$(shell id -g) $(CURDIR) -.PHONY: debug-jupyter-notebook +.PHONY: debug-jupyter-standalone debug-development: build-development-image ## Debug development environment @@ -467,7 +474,7 @@ test-unit-cpp-standalone: ## Run C++ unit tests (standalone) --workdir=/app/build \ $(docker_development_image_repository):$(docker_image_version) \ /bin/bash -c "cmake -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_UNIT_TESTS=ON .. \ - && $(MAKE) -j $(shell nproc) \ + && $(MAKE) -j 4 \ && $(MAKE) test" .PHONY: test-unit-cpp-standalone @@ -490,7 +497,7 @@ test-unit-python-standalone: ## Run Python unit tests (standalone) --entrypoint="" \ $(docker_development_image_repository):$(docker_image_version) \ /bin/bash -c "cmake -DBUILD_PYTHON_BINDINGS=ON -DBUILD_UNIT_TESTS=OFF .. \ - && $(MAKE) -j $(shell nproc) && python3.11 -m pip install --root-user-action=ignore bindings/python/dist/*311*.whl \ + && $(MAKE) -j 4 && python3.11 -m pip install --root-user-action=ignore bindings/python/dist/*311*.whl \ && python3.11 -m pip install plotly pandas \ && python3.11 -m pip install git+https://github.com/lucas-bremond/cesiumpy.git#egg=cesiumpy \ && cd /usr/local/lib/python3.11/site-packages/ostk/$(project_name)/ \ @@ -523,7 +530,7 @@ test-coverage-cpp-standalone: ## Run C++ tests with coverage (standalone) --workdir=/app/build \ $(docker_development_image_repository):$(docker_image_version) \ /bin/bash -c "cmake -DBUILD_UNIT_TESTS=ON -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_CODE_COVERAGE=ON .. \ - && $(MAKE) -j $(shell nproc) \ + && $(MAKE) -j 4 \ && $(MAKE) coverage \ && (rm -rf /app/coverage || true) \ && mkdir /app/coverage \ @@ -556,7 +563,7 @@ benchmark-cpp-standalone: ## Run C++ benchmarks (standalone) --workdir=/app/build \ $(docker_development_image_repository):$(docker_image_version) \ /bin/bash -c "cmake -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_UNIT_TESTS=OFF -DBUILD_BENCHMARK=ON .. \ - && $(MAKE) -j $(shell nproc) \ + && $(MAKE) -j 4 \ && ./../bin/open-space-toolkit-$(project_name).benchmark --benchmark_out_format=json --benchmark_out=./../bin/benchmark_result.json" .PHONY: benchmark-cpp-standalone From 5241fd2fd802614d35d0ce92dfc6d90be610ec44 Mon Sep 17 00:00:00 2001 From: Antoine Paletta <98616558+apaletta3@users.noreply.github.com> Date: Mon, 27 Nov 2023 18:54:52 +0100 Subject: [PATCH 5/6] fix: update Makefile with vishwa's suggestions Co-authored-by: Vishwa Shah --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index efa6655bc..b02ab10fd 100644 --- a/Makefile +++ b/Makefile @@ -307,7 +307,7 @@ debug-jupyter: build-development-image ## Debug jupyter notebook using the ostk- .PHONY: debug-jupyter -debug-jupyter-standalone: build-release-image-jupyter ## Debug jupyter notebook using the ostk-astro package built from current source code +debug-jupyter-rebuild: build-release-image-jupyter ## Debug jupyter notebook using the ostk-astro package built from current source code @ echo "Debugging Jupyter Notebook environment..." From c8623255eb004339de1ddc790c6283dba42fd9e9 Mon Sep 17 00:00:00 2001 From: Antoine Paletta <98616558+apaletta3@users.noreply.github.com> Date: Mon, 27 Nov 2023 18:55:12 +0100 Subject: [PATCH 6/6] Update Makefile Co-authored-by: Vishwa Shah --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b02ab10fd..11f02eef1 100644 --- a/Makefile +++ b/Makefile @@ -325,7 +325,7 @@ debug-jupyter-rebuild: build-release-image-jupyter ## Debug jupyter notebook usi @ sudo chown -R $(shell id -u):$(shell id -g) $(CURDIR) -.PHONY: debug-jupyter-standalone +.PHONY: debug-jupyter-rebuild debug-development: build-development-image ## Debug development environment