From b79e713c98ab4b3e543dbeaf13a48ae19213b176 Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Fri, 4 Oct 2024 13:29:24 +0200 Subject: [PATCH 01/15] Add virtual evn to developer scripts --- script/base.sh | 3 ++- script/env.sh | 20 ++++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/script/base.sh b/script/base.sh index 5912f1a39..ac44cf47f 100755 --- a/script/base.sh +++ b/script/base.sh @@ -14,7 +14,8 @@ sudo apt-get install -y \ apache2-utils \ net-tools \ python3-dev \ - python3-pip + python3-pip \ + python3-env sudo mkdir -p /etc/apt/keyrings if [ ! -f /etc/apt/keyrings/docker.gpg ] diff --git a/script/env.sh b/script/env.sh index 2c1e71b16..df7691efa 100755 --- a/script/env.sh +++ b/script/env.sh @@ -11,12 +11,20 @@ npx --yes playwright install-deps #------------- printf "\n\n Installing required python packages...." sudo apt install -y python3-pip -sudo -H pip3 install mkdocs -sudo -H pip3 install mkdocs-material -sudo -H pip3 install python-markdown-math -sudo -H pip3 install mkdocs-open-in-new-tab -sudo -H pip3 install mkdocs-with-pdf -sudo -H pip3 install qrcode + +python3 -m venv ./dtaas-venv + +chmod +x ./dtaas-venv/bin/activate +./dtaas-venv/bin/activate + +./dtaas-venv/bin/pip3 install mkdocs +./dtaas-venv/bin/pip3 install mkdocs-material +./dtaas-venv/bin/pip3 install python-markdown-math +./dtaas-venv/bin/pip3 install mkdocs-open-in-new-tab +./dtaas-venv/bin/pip3 install mkdocs-with-pdf +./dtaas-venv/bin/pip3 install qrcode + +deactivate # Install markdownlint sudo apt-get install -y rubygems From 69924f3e29f9a25f86c4eb41ed671323c3158531 Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Fri, 4 Oct 2024 13:29:47 +0200 Subject: [PATCH 02/15] Python env stop in development guide --- docs/developer/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/developer/index.md b/docs/developer/index.md index 3341797e8..e2ca35a9b 100644 --- a/docs/developer/index.md +++ b/docs/developer/index.md @@ -124,6 +124,7 @@ a development workflow is in place. Each developer should follow these steps: 1. Use [Fork, Branch, PR](https://gun.io/news/2017/01/how-to-github-fork-branch-and-pull-request/) workflow. +1. Active the Python Virtual Enviornment by calling (`./dtaas-venv/bin/activate`) 1. Work in your fork and open a PR from your working branch to your `feature/distributed-demo` branch. The PR will run all the github actions, code climate and codecov checks. From 387b2e0d0d966b4fc9c9d6da6f0c40b8ee655d7f Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Wed, 9 Oct 2024 12:50:46 +0200 Subject: [PATCH 03/15] Removed step for installing git-hooks. --- docs/developer/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/developer/index.md b/docs/developer/index.md index e2ca35a9b..be72574ca 100644 --- a/docs/developer/index.md +++ b/docs/developer/index.md @@ -120,7 +120,9 @@ a development workflow is in place. Each developer should follow these steps: [Codecov](https://docs.codecov.com/docs/quick-start) for your fork. The codecov does not require secret token for public repositories. + 1. Use [Fork, Branch, PR](https://gun.io/news/2017/01/how-to-github-fork-branch-and-pull-request/) workflow. From 42150c29e03d003cb6e93f5d260c02158279d128 Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Wed, 9 Oct 2024 12:55:23 +0200 Subject: [PATCH 04/15] code climate fix --- docs/developer/index.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/developer/index.md b/docs/developer/index.md index be72574ca..219f4ba59 100644 --- a/docs/developer/index.md +++ b/docs/developer/index.md @@ -113,6 +113,11 @@ option with care. To manage collaboration by multiple developers on the software, a development workflow is in place. Each developer should follow these steps: + + 1. Fork of the main repository into your github account. 1. Setup [Code Climate](https://docs.codeclimate.com/docs/getting-started-with-code-climate) @@ -120,9 +125,6 @@ a development workflow is in place. Each developer should follow these steps: [Codecov](https://docs.codecov.com/docs/quick-start) for your fork. The codecov does not require secret token for public repositories. - 1. Use [Fork, Branch, PR](https://gun.io/news/2017/01/how-to-github-fork-branch-and-pull-request/) workflow. From 1048b92b285167b390682ad6f01d2d9cd88dc1b1 Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Wed, 9 Oct 2024 12:55:23 +0200 Subject: [PATCH 05/15] Code climate fix --- docs/developer/index.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/developer/index.md b/docs/developer/index.md index be72574ca..99e0547d4 100644 --- a/docs/developer/index.md +++ b/docs/developer/index.md @@ -113,6 +113,11 @@ option with care. To manage collaboration by multiple developers on the software, a development workflow is in place. Each developer should follow these steps: + + 1. Fork of the main repository into your github account. 1. Setup [Code Climate](https://docs.codeclimate.com/docs/getting-started-with-code-climate) @@ -120,9 +125,6 @@ a development workflow is in place. Each developer should follow these steps: [Codecov](https://docs.codecov.com/docs/quick-start) for your fork. The codecov does not require secret token for public repositories. - 1. Use [Fork, Branch, PR](https://gun.io/news/2017/01/how-to-github-fork-branch-and-pull-request/) workflow. From 075eee20174f31590a28eb88ec5d3862035a27be Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Fri, 11 Oct 2024 09:08:27 +0200 Subject: [PATCH 06/15] Removed list from comment. --- docs/developer/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer/index.md b/docs/developer/index.md index 99e0547d4..c3c38973a 100644 --- a/docs/developer/index.md +++ b/docs/developer/index.md @@ -115,7 +115,7 @@ a development workflow is in place. Each developer should follow these steps: 1. Fork of the main repository into your github account. From 00db72d3ce649f4d9b55d6099bf61230b53492df Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Fri, 11 Oct 2024 14:43:04 +0200 Subject: [PATCH 07/15] Removed Docker compose, now included in docker --- script/base.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/script/base.sh b/script/base.sh index ac44cf47f..802ec3b5a 100755 --- a/script/base.sh +++ b/script/base.sh @@ -53,14 +53,6 @@ docker run hello-world sudo systemctl enable docker.service sudo systemctl enable containerd.service - -#install docker-compose from https://docs.docker.com/compose/install/other/ -sudo curl -SL "https://github.com/docker/compose/releases/download/v2.23.3/docker-compose-linux-x86_64" \ - -o /usr/local/bin/docker-compose -sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose -sudo chmod 755 /usr/local/bin/docker-compose /usr/bin/docker-compose - - # Install nodejs environment sudo apt-get update sudo apt-get install -y ca-certificates curl gnupg From 1ebfde16f4966992bb4584f11884bac563f8c9d4 Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Mon, 14 Oct 2024 15:28:43 +0200 Subject: [PATCH 08/15] Moved to nvm to handle node installation. --- docs/developer/index.md | 1 + script/base.sh | 16 ++++++---------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/docs/developer/index.md b/docs/developer/index.md index c3c38973a..988886c99 100644 --- a/docs/developer/index.md +++ b/docs/developer/index.md @@ -125,6 +125,7 @@ Install git-hooks for the project. [Codecov](https://docs.codecov.com/docs/quick-start) for your fork. The codecov does not require secret token for public repositories. +1. nvm use 22 (if nvm or node isn't installed) 1. Use [Fork, Branch, PR](https://gun.io/news/2017/01/how-to-github-fork-branch-and-pull-request/) workflow. diff --git a/script/base.sh b/script/base.sh index 802ec3b5a..93e936917 100755 --- a/script/base.sh +++ b/script/base.sh @@ -56,16 +56,12 @@ sudo systemctl enable containerd.service # Install nodejs environment sudo apt-get update sudo apt-get install -y ca-certificates curl gnupg -sudo mkdir -p /etc/apt/keyrings -if [ ! -f /etc/apt/keyrings/nodesource.gpg ] -then - curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | \ - sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg -fi -NODE_MAJOR=20 -printf "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] \ - https://deb.nodesource.com/node_%s.x nodistro main" "$NODE_MAJOR" | \ - sudo tee /etc/apt/sources.list.d/nodesource.list +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash +export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm + +nvm install 22 +nvm use 22 sudo apt-get update sudo apt-get install -y nodejs From d3d7d0848c0308d017e1f6c5126363ac931f1416 Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Mon, 14 Oct 2024 15:39:00 +0200 Subject: [PATCH 09/15] Added to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index bf75d60a4..93e4b0e3e 100644 --- a/.gitignore +++ b/.gitignore @@ -165,3 +165,6 @@ runner.yml # command scripts for runner servers/execution/runner/lifecycle* + +# Python virtual-env +dtaas-venv/ From d7ac91af19935392984df0980b153cd41d64a4f8 Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Wed, 23 Oct 2024 12:34:53 +0200 Subject: [PATCH 10/15] Removed git hooks section from developer doc --- docs/developer/index.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/developer/index.md b/docs/developer/index.md index 988886c99..d82ba72fc 100644 --- a/docs/developer/index.md +++ b/docs/developer/index.md @@ -76,6 +76,9 @@ powershell -F script/docker.ps1 about 5GB of bandwidth and 15GB of space. You will have to download the docker images on a really good network. + + ## :building_construction: Development Workflow To manage collaboration by multiple developers on the software, From 094e8b86fd0549046b01914e0f47d3176661e708 Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Wed, 23 Oct 2024 13:12:55 +0200 Subject: [PATCH 11/15] Updated docker installation to include buildx --- script/base.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/base.sh b/script/base.sh index 93e936917..344cd33d5 100755 --- a/script/base.sh +++ b/script/base.sh @@ -27,7 +27,7 @@ then fi sudo apt-get update -y -sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin +sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin sudo groupadd docker || true sudo usermod -aG docker "$USER" || true From 4e47d7cca18850a0ca67c943e0f667263232624e Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Thu, 7 Nov 2024 13:31:31 +0100 Subject: [PATCH 12/15] Python venv Create and use virtual env in windows for pip packages --- script/env.ps1 | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/script/env.ps1 b/script/env.ps1 index f9b368438..ee961422a 100644 --- a/script/env.ps1 +++ b/script/env.ps1 @@ -21,9 +21,6 @@ if (-not ([Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarg [Environment]::SetEnvironmentVariable("Path", "$env:Path;$(Get-Command python3 | Select-Object -ExpandProperty Directory)", [EnvironmentVariableTarget]::Machine) } -# Install mkdocs -pip install mkdocs - # Install Ruby: https://github.com/oneclick/rubyinstaller2/releases/download/RubyInstaller-3.1.2-1/rubyinstaller-devkit-3.1.2-1-x64.exe choco install -y ruby @@ -36,9 +33,18 @@ if (-not ([Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarg # Install markdownlint gem install mdl +python -m venv ./dtaas-venv +./dtaas-venv/Scripts/Activate + +# Install mkdocs +pip install mkdocs + # Install mkdocs plugins pip install mkdocs-material python-markdown-math mkdocs-open-in-new-tab mkdocs-with-pdf qrcode +# Deativate venv for python +deactivate + # Ensure Python scripts directory is added to PATH (for pip) if (-not ([Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) -like "*Python3*")) { Write-Host "Adding Python scripts to PATH (for pip)..." From e4e246d30f5747a3c6eccf613b9f6a27ef1e172d Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Thu, 7 Nov 2024 13:32:43 +0100 Subject: [PATCH 13/15] docker-compose and nvm Docker Compose is bundled in docker desktop NVM is used to update and install node. --- script/base.ps1 | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/script/base.ps1 b/script/base.ps1 index 47e218752..07fc2bac5 100644 --- a/script/base.ps1 +++ b/script/base.ps1 @@ -28,6 +28,10 @@ if (-not (Get-Command docker-desktop -ErrorAction SilentlyContinue)) { # https://docs.docker.com/engine/install/ubuntu/ choco install -y docker-desktop + $dockerDesktopPath = Join-Path $env:ProgramFiles "Docker\Docker\resources\cli-plugins" + # Ensure docker-desktop is in the system path + [Environment]::SetEnvironmentVariable("Path", $env:Path + ";$dockerDesktopPath", [EnvironmentVariableTarget]::Machine) + # Check if docker group exists, if not, create it $dockerGroup = 'docker' if (-not (Get-LocalGroup -Name $dockerGroup -ErrorAction SilentlyContinue)) { @@ -61,21 +65,22 @@ if (-not (Get-Command docker-desktop -ErrorAction SilentlyContinue)) { # Enable Docker service Set-Service -Name "com.docker.service" -StartupType Automatic - - # Install docker-compose from https://docs.docker.com/compose/install/other/ - $url = "https://github.com/docker/compose/releases/download/v2.23.3/docker-compose-windows-x86_64.exe" - $outputPath = Join-Path $env:ProgramFiles "Docker\docker-compose.exe" - Invoke-WebRequest -Uri $url -OutFile $outputPath - # Ensure docker-compose is in the system path - [Environment]::SetEnvironmentVariable("Path", $env:Path + ";$outputPath", [EnvironmentVariableTarget]::Machine) } # Check if Node.js is already installed if (-not (Get-Command node -ErrorAction SilentlyContinue)) { # Install Node.js environment if not already installed - choco install -y nodejs-lts + choco install -y nvm + $nvmInstallPath = Join-Path "C:\ProgramData" "nvm" + [Environment]::SetEnvironmentVariable("Path", $env:Path + ";$nvmInstallPath", [EnvironmentVariableTarget]::Machine) + Write-Host "installing nvm" } +# Install node +nvm install 22 +nvm use 22 + + # Install Yarn choco install -y yarn From ddf8e594ba9e7b50172d2a2c33d5394347192128 Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Thu, 7 Nov 2024 15:13:02 +0100 Subject: [PATCH 14/15] Type in apt-get --- script/base.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/base.sh b/script/base.sh index 344cd33d5..e19afbee5 100755 --- a/script/base.sh +++ b/script/base.sh @@ -15,7 +15,7 @@ sudo apt-get install -y \ net-tools \ python3-dev \ python3-pip \ - python3-env + python3-venv sudo mkdir -p /etc/apt/keyrings if [ ! -f /etc/apt/keyrings/docker.gpg ] @@ -79,4 +79,4 @@ fi sudo apt-get update -y sudo apt-get install -y yarn sudo npm install -g serve -sudo npm install -g pm2 \ No newline at end of file +sudo npm install -g pm2 From bda29330c03901f78beb61a67b3b9720fac78234 Mon Sep 17 00:00:00 2001 From: Henrik Kirk Date: Fri, 8 Nov 2024 08:55:25 +0100 Subject: [PATCH 15/15] Removed mention of git-hooks in dev index --- docs/developer/index.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/developer/index.md b/docs/developer/index.md index d82ba72fc..243f67efc 100644 --- a/docs/developer/index.md +++ b/docs/developer/index.md @@ -21,7 +21,6 @@ Please use the steps given here to install the required software packages. ```bash bash script/env.sh -bash script/configure-git-hooks.sh bash script/docker.sh ``` @@ -68,7 +67,6 @@ following commands in the given order: ```bash powershell -F script/base.ps1 powershell -F script/env.ps1 -powershell -F script/configure-git-hooks.ps1 powershell -F script/docker.ps1 ```